feat(cli): S4 follow-up, refusal backoff and tracker boot (row 45, #1527)

Rocko's round 2 candidate, packet agents/rocko/work/s4-follow-up/ (build.patch c8cec070, candidate manifest 5b067a9d, 8/8 OK). - Definite DM refusals wait the full 30-minute cap, counted from the journal's last refusal, so five refusals span about two hours before gave-up (lead decision 73). Unknown outcomes keep doubling. - Broker close sends at host.mjs:144/150/184/188 pass a callback, which closes the EPIPE window both reviewers found in round 1. - README documents manual recovery for an open decision. - trackers-boot test, X9, X14, and Darkwing's round 1 notes 1-4. - The append type check stays out; Rocko's reason holds (both reviewers agree). Reviews: Darkwing approve (comment 26884), Filbert approve (26886). Landing gate on b13fef4c plus the patch: every node suite and every scripts/test-*.sh green, test-task 98/0. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-09 09:27:34 -05:00
parent f4714aa03b
commit 9cdb6d82e3
8 changed files with 635 additions and 56 deletions
@@ -141,13 +141,13 @@ export async function startHost({ boot, business, notifier = null, bootTimeoutMs
    } catch (e) {
      notify.kill("SIGTERM");
      await ended(notify, 5000);
-      broker.send({ op: "close" });
+      if (broker.connected) broker.send({ op: "close" }, () => {});
      await ended(broker, CLOSE_TIMEOUT_MS);
      throw e;
    }
    if (ok?.ok !== true) {
      await ended(notify, 5000);
-      broker.send({ op: "close" });
+      if (broker.connected) broker.send({ op: "close" }, () => {});
      await ended(broker, CLOSE_TIMEOUT_MS);
      throw new CliError(`notifier refused to start: ${typeof ok?.error === "string" ? ok.error : "notifier-refused"}`, 3);
    }
@@ -181,11 +181,11 @@ export async function startHost({ boot, business, notifier = null, bootTimeoutMs
    closing = true;
    let result = code;
    if (notify) {
-      if (notify.connected) notify.send({ op: "stop" });
+      if (notify.connected) notify.send({ op: "stop" }, () => {});
      if ((await ended(notify, CLOSE_TIMEOUT_MS)) !== 0 && result === 0) result = 1;
    }
    await queue;
-    if (broker.connected) broker.send({ op: "close" });
+    if (broker.connected) broker.send({ op: "close" }, () => {});
    if ((await ended(broker, CLOSE_TIMEOUT_MS)) !== 0 && result === 0) result = 1;
    const now = readHostState(dataRoot);
    if (now && now.pid === state.pid && now.startTime === state.startTime) rmSync(hostFile(dataRoot), { force: true });
@@ -4,17 +4,25 @@
 // bus; its memory is the journal `<dataRoot>/notify/<business>/sent.jsonl`
 // (0600, in a 0700 directory), one line per send attempt:
 //
-//   {at, kind: "dm"|"digest", decision, day?, outcome: "confirmed"|"refused"|"unknown", messageId, status?}
+//   {at, kind: "dm"|"digest", decision, day?, outcome: "confirmed"|"refused"|"unknown"|"gave-up", messageId, status?}
 //
 // A decision counts as sent once it has a confirmed line; a day's digest
 // likewise. A refused or unknown send is retried with backoff (30 s
 // doubling to 30 min): a duplicate costs less than a miss, and Discord's
 // nonce folds a retry inside its dedupe window into the first message.
+// The exception is a definite refusal, an HTTP 4xx other than 429 (lead
+// decision 72): the next try waits the full 30 min, counted from the
+// refusal's line so a restart cannot shorten it (lead decision 73), and
+// after five for one decision, counted from the journal, the notifier
+// appends one `gave-up` line, logs once and stops sending that DM. Five
+// tries span two hours, long enough to fix a binding or token typo. The
+// digest marks it "DM refused, not retried"; recovery is the operator
+// deciding it. Unknown outcomes (network, 5xx, 429) retry without a limit.
 // No Discord channel or user id goes in the journal, a log line or an
 // error; the Discord side (packages/discord/src/notify.mjs) keeps them.

 import { createHash } from "node:crypto";
-import { closeSync, constants, fstatSync, fsyncSync, ftruncateSync, lstatSync, mkdirSync, openSync, readFileSync, writeSync } from "node:fs";
+import { accessSync, closeSync, constants, fstatSync, fsyncSync, ftruncateSync, lstatSync, mkdirSync, openSync, readFileSync, writeSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { CliError } from "./errors.mjs";
 import { authorizationLines, optionsLine, shortId } from "./format.mjs";
@@ -25,6 +33,7 @@ export const POLL_MS = 30000;
 const BACKOFF_MS = 30000;
 const BACKOFF_MAX_MS = 30 * 60 * 1000;
 const LIMIT = 2000;
+export const REFUSAL_LIMIT = 5;

 export const journalPath = (dataRoot, business) => join(dataRoot, "notify", business, "sent.jsonl");

@@ -62,14 +71,14 @@ export function dmContent(business, d) {
  return clip(lines.join("\n"), LIMIT);
 }

-export function digestContent(business, day, inbox, dmSent) {
+export function digestContent(business, day, inbox, dmSent, dmGaveUp = () => false) {
  if (inbox.length === 0) return `Mosaic digest (${business}, ${day}): your inbox is empty.`;
  const head = `Mosaic digest (${business}, ${day}): ${inbox.length} open decision(s).`;
  const tail = "Run mosaic inbox for the full list.";
  const lines = [head];
  let shown = 0;
  for (const d of inbox) {
-    const mark = d.blocking ? (dmSent(d.id) ? "[blocking, DM sent] " : "[blocking, DM pending] ") : "";
+    const mark = d.blocking ? (dmSent(d.id) ? "[blocking, DM sent] " : dmGaveUp(d.id) ? "[blocking, DM refused, not retried] " : "[blocking, DM pending] ") : "";
    const line = `- ${mark}${shortId(d.id)} ${d.action}: ${clip(d.question.replace(/\s+/g, " "), 160)}`;
    const more = inbox.length - shown - 1;
    const reserve = more > 0 ? `\n… and ${more} more.`.length : 0;
@@ -116,9 +125,32 @@ function copyTorn(dir, bytes, date) {
  }
 }

+const OUTCOMES = ["confirmed", "refused", "unknown", "gave-up"];
+const DAY = /^\d{4}-\d{2}-\d{2}$/;
+const isDay = (d) => typeof d === "string" && DAY.test(d) && !Number.isNaN(Date.parse(d)) && new Date(d).toISOString().startsWith(d);
+const UNWRITABLE = ["EACCES", "EPERM", "EROFS"];
+
+// The first field of a journal record that has the wrong type, or null
+// (lead decision 72, J4). A confirmed line carries the message id.
+function badField(r) {
+  if (!r || typeof r !== "object" || Array.isArray(r)) return "record";
+  if (typeof r.at !== "string" || Number.isNaN(Date.parse(r.at)) || new Date(r.at).toISOString() !== r.at) return "at";
+  if (!["dm", "digest"].includes(r.kind)) return "kind";
+  if (r.kind === "dm" ? typeof r.decision !== "string" || r.decision === "" : r.decision !== null && r.decision !== undefined) return "decision";
+  if (r.kind === "digest" ? !isDay(r.day) : r.day !== undefined && !isDay(r.day)) return "day";
+  if (!OUTCOMES.includes(r.outcome) || (r.outcome === "gave-up" && r.kind !== "dm")) return "outcome";
+  if (r.outcome === "confirmed" ? typeof r.messageId !== "string" : r.messageId !== null && typeof r.messageId !== "string") return "messageId";
+  if (r.status !== undefined && !Number.isInteger(r.status)) return "status";
+  return null;
+}
+
+// A DM refused with an HTTP 4xx other than 429.
+const definite = (r) => r.kind === "dm" && r.outcome === "refused" && Number.isInteger(r.status) && r.status >= 400 && r.status < 500 && r.status !== 429;
+
 // Opens (creating if needed) the journal. The directory must be 0700 or
-// tighter and the file 0600, both owned by this user; a symlinked journal
-// refuses. Every complete line must parse, or the open refuses with exit 3.
+// tighter, writable and not a symlink, and the file 0600, both owned by
+// this user; a symlinked journal refuses. Every complete line must parse
+// and type-check, or the open refuses with exit 3.
 // A final line without its newline is a write that never finished (lead
 // decision 71): its bytes are copied to torn-<stamp>.bin, then the journal
 // is truncated to its last newline and fsynced, and both steps are logged.
@@ -126,20 +158,49 @@ function copyTorn(dir, bytes, date) {
 // both; the second copy is harmless.
 export function openJournal(file, { log = () => {}, now = () => new Date() } = {}) {
  const dir = dirname(file);
-  mkdirSync(dir, { recursive: true, mode: 0o700 });
+  try {
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+  } catch (e) {
+    // A dangling link in place of the directory fails mkdir with ENOENT.
+    if (e.code === "ENOENT" && lstatSync(dir, { throwIfNoEntry: false })?.isSymbolicLink()) throw new CliError(`notify journal directory must not be a symlink: ${dir}`, 3);
+    if ([...UNWRITABLE, "ENOTDIR"].includes(e.code)) throw new CliError(`notify journal directory cannot be created (${e.code}): ${dir}`, 3);
+    if (e.code !== "EEXIST") throw e;
+  }
  const ds = lstatSync(dir);
+  if (ds.isSymbolicLink()) throw new CliError(`notify journal directory must not be a symlink: ${dir}`, 3);
  if (!ds.isDirectory() || ds.uid !== process.getuid() || (ds.mode & 0o077) !== 0) {
    throw new CliError(`notify journal directory must be mode 0700 and owned by this user: ${dir}`, 3);
  }
+  try {
+    accessSync(dir, constants.W_OK | constants.X_OK);
+  } catch (e) {
+    if (UNWRITABLE.includes(e.code)) throw new CliError(`notify journal directory is not writable (${e.code}): ${dir}`, 3);
+    throw e;
+  }
  let fd;
  try {
    fd = openSync(file, O_RDWR | O_APPEND | O_CREAT | O_NOFOLLOW, 0o600);
  } catch (e) {
    if (e.code === "ELOOP") throw new CliError(`notify journal must not be a symlink: ${file}`, 3);
+    if (UNWRITABLE.includes(e.code)) throw new CliError(`notify journal is not writable (${e.code}): ${file}`, 3);
+    if (e.code === "EISDIR") throw new CliError(`notify journal must be a regular file, mode 0600, owned by this user: ${file}`, 3);
    throw e;
  }
  const sent = new Set();
  const days = new Set();
+  const refusals = new Map();
+  const gaveUp = new Set();
+  const refusedAt = new Map();
+  const count = (r) => {
+    if (r.kind === "dm" && r.outcome === "gave-up") gaveUp.add(r.decision);
+    if (definite(r)) {
+      refusals.set(r.decision, (refusals.get(r.decision) ?? 0) + 1);
+      refusedAt.set(r.decision, Date.parse(r.at));
+    }
+    if (r.outcome !== "confirmed") return;
+    if (r.kind === "dm") sent.add(r.decision);
+    else days.add(r.day);
+  };
  try {
    const st = fstatSync(fd);
    if (!st.isFile() || st.uid !== process.getuid() || (st.mode & 0o777) !== 0o600) {
@@ -156,12 +217,9 @@ export function openJournal(file, { log = () => {}, now = () => new Date() } = {
      } catch {
        r = null;
      }
-      if (!r || typeof r !== "object" || !["dm", "digest"].includes(r.kind)) {
-        throw new CliError(`notify journal line ${i + 1} is malformed: ${file}`, 3);
-      }
-      if (r.outcome !== "confirmed") return;
-      if (r.kind === "dm") sent.add(r.decision);
-      else days.add(r.day);
+      const bad = r === null ? "json" : badField(r);
+      if (bad) throw new CliError(`notify journal line ${i + 1} is malformed (${bad}): ${file}`, 3);
+      count(r);
    });
    if (end < bytes.length) {
      const name = copyTorn(dir, bytes.subarray(end), now());
@@ -176,6 +234,9 @@ export function openJournal(file, { log = () => {}, now = () => new Date() } = {
  return {
    sent,
    days,
+    refusals,
+    gaveUp,
+    refusedAt,
    append(record) {
      const afd = openSync(file, O_WRONLY | O_APPEND | O_NOFOLLOW);
      try {
@@ -183,9 +244,7 @@ export function openJournal(file, { log = () => {}, now = () => new Date() } = {
      } finally {
        closeSync(afd);
      }
-      if (record.outcome !== "confirmed") return;
-      if (record.kind === "dm") sent.add(record.decision);
-      else days.add(record.day);
+      count(record);
    },
  };
 }
@@ -202,6 +261,13 @@ export function createNotifier({ business, dataRoot, inbox, direct, now = () =>
    return b !== undefined && t < b.next;
  }

+  // One `gave-up` line and one log line; the decision is never sent again.
+  function giveUp(decision, t) {
+    journal.append({ at: new Date(t).toISOString(), kind: "dm", decision, outcome: "gave-up", messageId: null });
+    backoff.delete(`dm:${decision}`);
+    log(`notify: dm ${shortId(decision)} refused ${journal.refusals.get(decision)} times; gave up, not retried`);
+  }
+
  async function attempt(key, record, message) {
    const t = now().getTime();
    try {
@@ -212,10 +278,16 @@ export function createNotifier({ business, dataRoot, inbox, direct, now = () =>
    } catch (err) {
      const kind = err?.kind === "refused" ? "refused" : "unknown";
      const status = Number.isInteger(err?.details?.status) ? err.details.status : null;
+      const line = { at: new Date(t).toISOString(), ...record, outcome: kind, messageId: null, ...(status !== null ? { status } : {}) };
      const n = (backoff.get(key)?.n ?? -1) + 1;
-      backoff.set(key, { n, next: t + Math.min(BACKOFF_MS * 2 ** n, BACKOFF_MAX_MS) });
-      journal.append({ at: new Date(t).toISOString(), ...record, outcome: kind, messageId: null, ...(status !== null ? { status } : {}) });
-      log(`notify: ${record.kind} ${kind}${status !== null ? ` (HTTP ${status})` : ""}; retry in ${Math.round(Math.min(BACKOFF_MS * 2 ** n, BACKOFF_MAX_MS) / 1000)} s`);
+      const wait = definite(line) ? BACKOFF_MAX_MS : Math.min(BACKOFF_MS * 2 ** n, BACKOFF_MAX_MS);
+      backoff.set(key, { n, next: t + wait });
+      journal.append(line);
+      if (record.kind === "dm" && (journal.refusals.get(record.decision) ?? 0) >= REFUSAL_LIMIT) {
+        giveUp(record.decision, t);
+        return false;
+      }
+      log(`notify: ${record.kind} ${kind}${status !== null ? ` (HTTP ${status})` : ""}; retry in ${Math.round(wait / 1000)} s`);
      return false;
    }
  }
@@ -232,7 +304,15 @@ export function createNotifier({ business, dataRoot, inbox, direct, now = () =>
    }
    const t = now();
    for (const d of list) {
-      if (!d.blocking || journal.sent.has(d.id)) continue;
+      if (!d.blocking || journal.sent.has(d.id) || journal.gaveUp.has(d.id)) continue;
+      // A crash between the fifth refusal and its gave-up line.
+      if ((journal.refusals.get(d.id) ?? 0) >= REFUSAL_LIMIT) {
+        giveUp(d.id, t.getTime());
+        continue;
+      }
+      // A definite refusal waits the full cap from its journal line, which
+      // the in-memory backoff loses on a restart.
+      if (t.getTime() < (journal.refusedAt.get(d.id) ?? -Infinity) + BACKOFF_MAX_MS) continue;
      const key = `dm:${d.id}`;
      if (waiting(key, t.getTime())) continue;
      if (await attempt(key, { kind: "dm", decision: d.id }, { content: dmContent(business, d), nonce: dmNonce(d.id) })) done.dms++;
@@ -241,7 +321,7 @@ export function createNotifier({ business, dataRoot, inbox, direct, now = () =>
    const { day, hour } = zoned(t, zone);
    const key = `digest:${day}`;
    if (hour >= DIGEST_HOUR && !journal.days.has(day) && !waiting(key, t.getTime())) {
-      const content = digestContent(business, day, list, (id) => journal.sent.has(id));
+      const content = digestContent(business, day, list, (id) => journal.sent.has(id), (id) => journal.gaveUp.has(id));
      if (await attempt(key, { kind: "digest", decision: null, day }, { content, nonce: digestNonce(business, day) })) done.digest = true;
      else done.failed++;
    }