feat(discord): systemd user service with a supervised run; brakes exit 3 and are never retried (#1509)
QUEUE row 17, MVP iteration 2. scripts/discord-service.sh renders and installs mosaic-discord@<binding> from packages/discord/systemd/. The unit's main process is `run --supervised`, which applies the new recover policy first: a lock whose owner is gone is cleared and only the STOP written for that is removed; an operator STOP or a held binding refuses with exit 3, which RestartPreventExitStatus never retries. `recover` is also a CLI verb. First cut used ExecStartPre and looped live, since systemd honours the never-retry status only from the main process; replaced and re-verified before any message traffic. Suite 40/40, 95 node tests. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Claude Fable 5.1
@@ -137,6 +137,6 @@ test("cli: run refuses when STOP is present, before any network use", () => {
|
||||
assert.equal(r0.status, 0, r0.stderr);
|
||||
assert.match(r0.stdout, /STOP written/);
|
||||
const r = runCli(["run", "test-seat", "--config", dep.config, "--repo", repo]);
|
||||
assert.equal(r.status, 1, r.stderr);
|
||||
assert.equal(r.status, 3, r.stderr);
|
||||
assert.match(r.stderr, /STOP is present/);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
// `recover`: the supervised pre-start behind the service unit. It may clear
|
||||
// a lock whose owner is gone and remove only the STOP it wrote for that;
|
||||
// every operator brake, and every held binding, refuses with exit 3.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { writeFileSync, existsSync, mkdirSync, readFileSync, unlinkSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
ensureJournal, recover, requestStop, readStop, stopRequested, stopPath, lockPath, ownerPath, writePid, BRAKE_EXIT,
|
||||
RECOVER_REASON,
|
||||
} from "../src/journal.mjs";
|
||||
import { DiscordError } from "../src/errors.mjs";
|
||||
import { makeRoot, makeRepo, makeDeployment } from "./helpers.mjs";
|
||||
|
||||
const cli = fileURLToPath(new URL("../src/cli.mjs", import.meta.url));
|
||||
const worker = fileURLToPath(new URL("../fixtures/legacy-owner-worker.mjs", import.meta.url));
|
||||
const BOOT = "01234567-89ab-cdef-0123-456789abcdef";
|
||||
|
||||
function journal() {
|
||||
const dir = join(makeRoot(), "journal");
|
||||
ensureJournal(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
function deadPid() {
|
||||
const r = spawnSync(process.execPath, ["-e", "process.stdout.write(String(process.pid))"], { encoding: "utf8" });
|
||||
return Number(r.stdout);
|
||||
}
|
||||
|
||||
function publish(dir, rec) {
|
||||
mkdirSync(lockPath(dir), { recursive: true });
|
||||
writeFileSync(ownerPath(dir), JSON.stringify(rec) + "\n", { mode: 0o600 });
|
||||
}
|
||||
|
||||
const brake = (re) => (err) => err instanceof DiscordError && err.exitCode === BRAKE_EXIT && re.test(err.message);
|
||||
|
||||
// A live owner with the given identity arguments, held until `release`.
|
||||
function holdLock(dir, startArg, bootArg) {
|
||||
const done = join(dir, `done-${startArg}-${bootArg}`);
|
||||
const child = spawn(process.execPath, [worker, dir, done, startArg, bootArg], { stdio: ["ignore", "pipe", "inherit"] });
|
||||
const published = new Promise((resolve) => {
|
||||
let out = "";
|
||||
child.stdout.on("data", (d) => { out += d; if (out.includes("legacy-published")) resolve(); });
|
||||
});
|
||||
const closed = new Promise((resolve) => child.on("close", resolve));
|
||||
return { published, release: () => { writeFileSync(done, ""); return closed; } };
|
||||
}
|
||||
|
||||
test("recover: nothing to do is clean; a lock whose owner is gone or that has no record is cleared and STOP ends up absent", () => {
|
||||
const dir = journal();
|
||||
assert.equal(recover(dir), "clean");
|
||||
assert.equal(stopRequested(dir), false);
|
||||
|
||||
publish(dir, { pid: deadPid(), start: "1", boot: BOOT, at: "x" });
|
||||
assert.equal(recover(dir), "cleared");
|
||||
assert.equal(existsSync(lockPath(dir)), false, "dead owner's lock removed");
|
||||
assert.equal(stopRequested(dir), false, "the STOP written for the unlock is gone again");
|
||||
|
||||
mkdirSync(lockPath(dir));
|
||||
assert.equal(recover(dir), "cleared", "a lock without a record (interrupted start) is cleared too");
|
||||
assert.equal(existsSync(lockPath(dir)), false);
|
||||
assert.equal(stopRequested(dir), false);
|
||||
assert.equal(recover(dir), "clean");
|
||||
});
|
||||
|
||||
test("recover: an operator STOP refuses with exit 3 and is never removed, whatever the lock says", () => {
|
||||
const dir = journal();
|
||||
requestStop(dir, "cli stop");
|
||||
assert.throws(() => recover(dir), brake(/STOP is present/));
|
||||
assert.equal(stopRequested(dir), true);
|
||||
publish(dir, { pid: deadPid(), start: "1", boot: BOOT, at: "x" });
|
||||
assert.throws(() => recover(dir), brake(/STOP is present/));
|
||||
assert.equal(existsSync(lockPath(dir)), true, "the lock is not touched behind a brake");
|
||||
assert.equal(readStop(dir).length, 1, "nothing appended to STOP");
|
||||
|
||||
// A STOP that recover itself wrote and then failed to remove is ours, but
|
||||
// only when it is the whole file.
|
||||
const own = journal();
|
||||
requestStop(own, RECOVER_REASON);
|
||||
assert.equal(recover(own), "clean");
|
||||
assert.equal(stopRequested(own), false);
|
||||
requestStop(own, RECOVER_REASON);
|
||||
requestStop(own, "unlock");
|
||||
assert.throws(() => recover(own), brake(/STOP is present/));
|
||||
assert.equal(readStop(own).length, 2);
|
||||
|
||||
// A line this code did not write is never ours.
|
||||
const foreign = journal();
|
||||
writeFileSync(stopPath(foreign), "stop\n", { mode: 0o600 });
|
||||
assert.throws(() => recover(foreign), brake(/STOP is present/));
|
||||
assert.deepEqual(readStop(foreign), [{ reason: null }]);
|
||||
});
|
||||
|
||||
test("recover: a brake written during the unlock wins; STOP stays with both lines and the start is refused", () => {
|
||||
const dir = journal();
|
||||
publish(dir, { pid: deadPid(), start: "1", boot: BOOT, at: "x" });
|
||||
assert.throws(
|
||||
() => recover(dir, { beforeRemove: () => requestStop(dir, "cli stop") }),
|
||||
brake(/written by an operator while recovering/),
|
||||
);
|
||||
assert.equal(existsSync(lockPath(dir)), false, "the dead lock was removed before the brake was seen");
|
||||
assert.deepEqual(readStop(dir).map((l) => l.reason), [RECOVER_REASON, "cli stop"]);
|
||||
assert.throws(() => recover(dir), brake(/STOP is present/), "and it stays refused afterwards");
|
||||
});
|
||||
|
||||
test("recover: a held binding refuses with exit 3 and writes no STOP: live owner, alive pid without verifiable identity, unreadable record", async () => {
|
||||
const live = journal();
|
||||
const held = holdLock(live, "real", "real");
|
||||
await held.published;
|
||||
try {
|
||||
assert.throws(() => recover(live), brake(/another connector is running/));
|
||||
assert.equal(stopRequested(live), false, "recover never brakes a running connector");
|
||||
assert.equal(existsSync(ownerPath(live)), true);
|
||||
} finally {
|
||||
await held.release();
|
||||
}
|
||||
|
||||
const unknown = journal();
|
||||
const legacy = holdLock(unknown, "real", "-");
|
||||
await legacy.published;
|
||||
try {
|
||||
assert.throws(() => recover(unknown), brake(/cannot be verified/));
|
||||
assert.equal(stopRequested(unknown), false);
|
||||
assert.equal(existsSync(ownerPath(unknown)), true);
|
||||
} finally {
|
||||
await legacy.release();
|
||||
}
|
||||
|
||||
const invalid = journal();
|
||||
mkdirSync(lockPath(invalid));
|
||||
writeFileSync(ownerPath(invalid), "{not json", { mode: 0o600 });
|
||||
assert.throws(() => recover(invalid), brake(/cannot be read/));
|
||||
assert.equal(stopRequested(invalid), false);
|
||||
assert.equal(readFileSync(ownerPath(invalid), "utf8"), "{not json");
|
||||
|
||||
// Our own live claim is a held binding too.
|
||||
const mine = journal();
|
||||
writePid(mine, process.pid);
|
||||
assert.throws(() => recover(mine), brake(/another connector is running/));
|
||||
assert.equal(stopRequested(mine), false);
|
||||
});
|
||||
|
||||
test("cli: recover exits 0 when ready, 3 behind a brake or a held binding, and run's own STOP refusal is 3", () => {
|
||||
const root = makeRoot();
|
||||
const repo = makeRepo(root);
|
||||
const dep = makeDeployment(root);
|
||||
const run = (args) => spawnSync(process.execPath, [cli, ...args, "--config", dep.config, "--repo", repo], { encoding: "utf8" });
|
||||
let r = run(["recover", "test-seat"]);
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
assert.match(r.stdout, /ready to run/);
|
||||
|
||||
r = run(["stop", "test-seat"]);
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
r = run(["recover", "test-seat"]);
|
||||
assert.equal(r.status, 3, r.stderr);
|
||||
assert.match(r.stderr, /the brake is on/);
|
||||
r = run(["run", "test-seat"]);
|
||||
assert.equal(r.status, 3, r.stderr);
|
||||
|
||||
const journalDir = join(dep.dataRoot, "discord", "test-seat");
|
||||
unlinkSync(stopPath(journalDir));
|
||||
publish(journalDir, { pid: deadPid(), start: "1", boot: BOOT, at: "x" });
|
||||
r = run(["recover", "test-seat"]);
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
assert.match(r.stdout, /was removed; STOP is absent/);
|
||||
assert.equal(existsSync(lockPath(journalDir)), false);
|
||||
assert.equal(stopRequested(journalDir), false);
|
||||
|
||||
r = run(["recover"]);
|
||||
assert.equal(r.status, 4);
|
||||
|
||||
// The service unit's form: the same policy inside the main process, so
|
||||
// the exit status is the one systemd never retries.
|
||||
r = run(["stop", "test-seat"]);
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
r = run(["run", "test-seat", "--supervised"]);
|
||||
assert.equal(r.status, 3, r.stderr);
|
||||
assert.match(r.stderr, /the brake is on/);
|
||||
assert.equal(stopRequested(journalDir), true, "the operator's STOP stays");
|
||||
unlinkSync(stopPath(journalDir));
|
||||
publish(journalDir, { pid: process.pid, at: "x" });
|
||||
r = run(["run", "test-seat", "--supervised"]);
|
||||
assert.equal(r.status, 3, r.stderr);
|
||||
assert.match(r.stderr, /cannot be verified/);
|
||||
assert.equal(existsSync(ownerPath(journalDir)), true, "a held binding is left alone");
|
||||
assert.equal(stopRequested(journalDir), false);
|
||||
r = run(["stop", "test-seat", "--supervised"]);
|
||||
assert.equal(r.status, 4);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user