Files
stack/tools/unslop-hook/test-unslop-check.js
jason.woltje 2c38325eca row 66: managed sessions run the unslop check on both harnesses, the operator note on the RESULT, a correction notice on the next turn, and unslop.jsonl counts (#1559)
Lands Sage's sage/row66-r5 at 7ffb640ac, round 4 (r4d 82c830779) rebased onto a36523713 after row 197,
approved by Dewey (comment 30085) and Filbert (30086) on manifest 6cfd812da4ca (23 paths, rev 2905). The
rebase had no conflict. The range-diff is = except one pi-session.test.mjs hunk header. Patch-id without
agents/ is d67275ac4251 on 7b9737953, a36523713 and this commit's parent. Gate (mos-1186): unslop,
harness, cli and every scripts/test-*.sh green on r4d, harness 149/0 again on r5. Landing sessions on a
scratch root from the candidate tree: Claude Code r4ab24630d323 showed the note, the injected notice and
counts checked 2, flagged 1, notices 1. Pi r1dd94171bede is launch only until #383 (Vikunja #454).
2026-10-21 13:51:28 -05:00

310 lines
16 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"use strict";
// Tests for unslop-check.js. Run: node test-unslop-check.js
// Exit 0 = all pass. Cases include a red control (slop must fail) and a green
// control (clean prose must pass) per evidence discipline.
const assert = require("node:assert");
const fs = require("node:fs");
const os = require("node:os");
const path = require("node:path");
const { spawnSync } = require("node:child_process");
const { checkText, stripCode, loadLists } = require("./unslop-check.js");
const SLOP = `Certainly! Let me delve into the evolving tapestry of database technology — it’s truly “pivotal” — and intricate.
In order to understand it — we should leverage this interplay of systems — deeply. I hope this helps!`;
const CLEAN = `The loader parses the file and validates each row. Rows that fail are logged
and skipped. We measured a range from 1 to 10 seconds. Use "straight quotes" and
commas, not dashes. That is the whole finding.`;
// Code-stripping control: banned words inside code must not count.
const WITH_CODE = [
"The config uses `utilize=false` internally.",
"```",
"delve tapestry — pivotal",
"```",
"The config file sets one flag. It is parsed at startup.",
].join("\n");
const results = [];
function t(name, fn) {
try { fn(); results.push([name, true]); } catch (e) { results.push([name, false]); console.error(`FAIL ${name}: ${e.message}`); }
}
t("slop fixture is flagged (red control)", () => {
const r = checkText(SLOP);
assert.ok(!r.clean, "slop must not be clean");
const details = r.findings.map((f) => f.detail).join("; ");
assert.ok(r.findings.some((f) => f.detail.includes("delve")), `delve missing: ${details}`);
assert.ok(r.findings.some((f) => f.detail.includes("tapestry")), `tapestry missing: ${details}`);
assert.ok(r.findings.some((f) => f.detail.includes("pivotal")), `pivotal missing: ${details}`);
assert.ok(r.findings.some((f) => f.detail.includes("em dash")), `em dash missing: ${details}`);
assert.ok(r.findings.some((f) => f.detail.includes("curly")), `curly missing: ${details}`);
assert.ok(r.findings.some((f) => f.detail.includes("in order to")), `in order to missing: ${details}`);
assert.ok(r.findings.some((f) => f.detail.includes("i hope this helps")), `chatbot phrase missing: ${details}`);
assert.ok(r.findings.some((f) => f.detail.includes("certainly")), `certainly missing: ${details}`);
});
t("clean fixture passes (green control)", () => {
const r = checkText(CLEAN);
assert.deepStrictEqual(r.findings, [], `unexpected findings: ${JSON.stringify(r.findings)}`);
});
t("numeric range is not a false range flag", () => {
const r = checkText(CLEAN);
assert.ok(!r.findings.some((f) => f.rule === "pattern"), "must not flag numeric ranges");
});
t("code blocks and inline code are stripped", () => {
const r = checkText(WITH_CODE);
assert.deepStrictEqual(r.findings, [], `code leaked into check: ${JSON.stringify(r.findings)}`);
});
t("stripCode removes fenced and inline code", () => {
const s = stripCode("a `x — y` b\n```\ndelve\n```\nc");
assert.ok(!s.includes("delve"), "fenced code not stripped");
assert.ok(!s.includes("—"), "inline code not stripped");
assert.ok(s.includes("a") && s.includes("b") && s.includes("c"), "prose lost");
});
t("not-just-but pattern is detected", () => {
const r = checkText("This is not just a cache but a coordination layer.");
assert.ok(r.findings.some((f) => f.rule === "pattern"), "pattern missed");
});
t("light dash use is not flagged (below threshold)", () => {
const prose =
"The loader parses each row and validates it against the schema. Rows that fail " +
"are logged — with their line numbers — and skipped. The operator reviews the log " +
"daily and reconciles the rejects against the source system by hand, which takes " +
"a few minutes and has never once produced a discrepancy worth acting on.";
const r = checkText(prose);
assert.ok(!r.findings.some((f) => f.detail.includes("dash")), "2 dashes in ~330 chars must not flag");
});
t("dash overuse is flagged (above threshold)", () => {
const r = checkText("One — two — three — four. That is the whole sentence.");
assert.ok(r.findings.some((f) => f.detail.includes("em dash")), "4 dashes in 50 chars must flag");
});
// ── C1: lists.json machine source ──────────────────────────────────────
// The committed lists are the single source of truth; these tests pin the
// file's validity, its shape, and the loader's fail-closed behavior.
const tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), "unslop-c1-"));
const tmp = (n) => path.join(tmpdir, n);
function brokenVariant(mutate) {
const l = JSON.parse(JSON.stringify(loadLists()));
mutate(l);
return l;
}
function writeTmp(name, data) {
const f = tmp(name);
fs.writeFileSync(f, typeof data === "string" ? data : JSON.stringify(data));
return f;
}
t("lists.json (the real file) validates and is pinned in size", () => {
const l = loadLists();
assert.strictEqual(l.version, 1);
// Counts pin the migration: 16 words, 17 phrases, 3 punct, 1 pattern moved
// from the old inline constants. Changing a count means changing this test
// too, consciously.
assert.strictEqual(l.words.length, 16, "word count drifted");
assert.strictEqual(l.phrases.length, 17, "phrase count drifted");
assert.strictEqual(l.punct.length, 3, "punct count drifted");
assert.strictEqual(l.patterns.length, 1, "pattern count drifted");
assert.ok(l.convention.length > 50, "mention convention must be present");
assert.ok(l.punctPolicy.length > 50, "punct divergence policy must be present");
for (const e of [...l.words, ...l.phrases, ...l.punct, ...l.patterns]) {
assert.ok(e.source && e.source.trim(), `entry missing source: ${JSON.stringify(e)}`);
}
});
t("loader rejects an empty file", () => {
const f = writeTmp("empty.json", "");
assert.throws(() => loadLists(f), /empty file/);
});
t("loader rejects unparseable JSON", () => {
const f = writeTmp("bad.json", "{nope");
assert.throws(() => loadLists(f), /unparseable/);
});
t("loader rejects a missing file", () => {
assert.throws(() => loadLists(tmp("does-not-exist.json")), /cannot read/);
});
t("loader rejects missing keys", () => {
const f = writeTmp("nokeys.json", { version: 1 });
assert.throws(() => loadLists(f), /missing key/);
});
t("loader rejects an emptied word list", () => {
const f = writeTmp("emptywords.json", brokenVariant((l) => { l.words = []; }));
assert.throws(() => loadLists(f), /words must be a non-empty array/);
});
t("loader rejects entries without provenance", () => {
const f = writeTmp("nosource.json", brokenVariant((l) => { delete l.phrases[0].source; }));
assert.throws(() => loadLists(f), /source/);
});
t("loader rejects duplicate values", () => {
const f = writeTmp("dup.json", brokenVariant((l) => { l.words.push({ ...l.words[0] }); }));
assert.throws(() => loadLists(f), /duplicate/);
});
t("loader rejects a non-compiling pattern regex", () => {
const f = writeTmp("badregex.json", brokenVariant((l) => { l.patterns[0].regex = "("; }));
assert.throws(() => loadLists(f), /does not compile/);
});
t("CLI exits 2 on broken lists (red control)", () => {
const f = writeTmp("cli-broken.json", "");
const r = spawnSync(process.execPath, [path.join(__dirname, "unslop-check.js")], {
input: "some prose",
encoding: "utf8",
env: { ...process.env, UNSLOP_LISTS: f },
});
assert.strictEqual(r.status, 2, `expected exit 2, got ${r.status} (stderr: ${r.stderr})`);
assert.ok(r.stderr.includes("lists.json invalid"), `stderr must name the fault: ${r.stderr}`);
});
t("CLI honors UNSLOP_LISTS for a valid file (green control)", () => {
const r = spawnSync(process.execPath, [path.join(__dirname, "unslop-check.js")], {
input: "plain prose with no tells at all",
encoding: "utf8",
env: { ...process.env, UNSLOP_LISTS: path.join(__dirname, "lists.json") },
});
assert.strictEqual(r.status, 0, `expected exit 0, got ${r.status} (stderr: ${r.stderr})`);
});
// ── Review follow-up (rev-code-01, 2026-08-19): F1, F2, F3, S3 ────────
t("loader rejects non-finite thresholds (F1)", () => {
// Infinity cannot round-trip JSON.stringify, so the fixture is a raw string
// edit of the real file — exactly the hand-edit that produced the finding.
const real = fs.readFileSync(path.join(__dirname, "lists.json"), "utf8");
const f = writeTmp("inf-threshold.json", real.replace('"punctMinCount": 3', '"punctMinCount": 1e999'));
assert.ok(real !== fs.readFileSync(f, "utf8") || !real.includes('"punctMinCount": 3'), "fixture mutation did not apply; test is vacuous");
assert.throws(() => loadLists(f), /finite/);
const f2 = writeTmp("inf-density.json", real.replace('"punctDensityPer1k": 1.0', '"punctDensityPer1k": 1e999'));
assert.throws(() => loadLists(f2), /finite/);
});
t("curly-apostrophe phrase rendering is flagged (F2 red control)", () => {
const r = checkText("Here\u2019s the honest truth about the deploy.");
assert.ok(!r.clean, "curly apostrophe must not defeat phrase matching");
assert.ok(r.findings.some((x) => x.detail.includes("here's the honest truth")), `main entry must match, got: ${JSON.stringify(r.findings)}`);
});
t("curly apostrophes still fire the punct rule alongside phrases (F2 ordering)", () => {
// Normalization for phrases must not eat the punct signal: four curly
// quotes in short text must flag punct, not only the phrase.
const r = checkText("It\u2019s \u2019one\u2019 \u2019two\u2019 \u2019three\u2019 \u2019four\u2019 done.");
assert.ok(r.findings.some((f) => f.rule === "punct"), `punct must fire on original text: ${JSON.stringify(r.findings)}`);
});
t("loader rejects non-lowercase phrase values (F3)", () => {
const f = writeTmp("cap-phrase.json", brokenVariant((l) => { l.phrases[0].value = "Worth Stating Plainly"; }));
assert.throws(() => loadLists(f), /lowercase/);
});
t("CLI exits 2 on unreadable input file (S3)", () => {
const r = spawnSync(process.execPath, [path.join(__dirname, "unslop-check.js"), tmp("definitely-absent.txt")], {
encoding: "utf8",
});
assert.strictEqual(r.status, 2, `expected exit 2, got ${r.status} (stderr: ${r.stderr})`);
assert.ok(r.stderr.includes("cannot read input"), `stderr must name the fault: ${r.stderr}`);
});
t("every finding names its lists.json entry's source, the pattern id (row 66)", () => {
const lists = loadLists();
const sources = new Set([...lists.words, ...lists.phrases, ...lists.punct, ...lists.patterns].map((e) => e.source));
const r = checkText(SLOP + "\nIt's not just fast, but cheap.");
for (const rule of ["word", "phrase", "punct", "pattern"]) assert.ok(r.findings.some((f) => f.rule === rule), `no ${rule} finding: ${JSON.stringify(r.findings)}`);
for (const f of r.findings) assert.ok(sources.has(f.source), `finding without a known source: ${JSON.stringify(f)}`);
assert.strictEqual(r.findings.find((f) => f.detail.includes("delve")).source, "ms-unslop:7");
});
// ── Row 66 (#1559): claude-hook.js, the Claude Code side ───────────────
// The session tests (packages/harness/tests/claude-session.test.mjs) run it under real claude;
// these pin its parts without one.
const HOOK = path.join(__dirname, "claude-hook.js");
const { replies } = require("./claude-hook.js");
const line = (type, id, uuid, content) => ({ type, uuid, message: { id, role: type, content } });
t("claude-hook replies(): adjacent lines of one message join, a repeated message id later is a new reply", () => {
const r = replies([
line("assistant", "m1", "u1", [{ type: "text", text: "one" }]),
line("assistant", "m1", "u2", [{ type: "tool_use", id: "x", name: "Read", input: {} }]),
line("user", undefined, "u3", [{ type: "tool_result", tool_use_id: "x", content: "ok" }]),
line("assistant", "m1", "u4", [{ type: "text", text: "two" }]),
line("assistant", "m1", "u5", [{ type: "text", text: "three" }]),
line("assistant", "m2", "u6", [{ type: "tool_use", id: "y", name: "Read", input: {} }]),
]);
assert.deepStrictEqual(r, [{ id: "u1", text: "one" }, { id: "u4", text: "two\nthree" }]);
});
function hook(which, input, env = {}) {
return spawnSync(process.execPath, [HOOK, which], { input: typeof input === "string" ? input : JSON.stringify(input), encoding: "utf8", env: { PATH: process.env.PATH, ...env } });
}
const events = (f) => (fs.existsSync(f) ? fs.readFileSync(f, "utf8").trim().split("\n").filter(Boolean).map((l) => JSON.parse(l)) : []);
t("claude-hook: Stop logs the turn's reply flagged with pattern ids, the next prompt gets the notice once, and both exit 0", () => {
const transcript = writeTmp("transcript.jsonl", [
{ type: "user", uuid: "p1", message: { role: "user", content: "status?" } },
line("assistant", "m1", "a1", [{ type: "text", text: "Let me delve into it. A pivotal step." }]),
].map((x) => JSON.stringify(x)).join("\n") + "\n");
const log = tmp("hook-log.jsonl");
const stop = hook("stop", { transcript_path: transcript }, { MOSAIC_UNSLOP_LOG: log });
assert.strictEqual(stop.status, 0, stop.stderr);
assert.strictEqual(stop.stdout, "");
const [flagged] = events(log);
assert.strictEqual(flagged.ev, "flagged");
assert.strictEqual(flagged.entryId, "a1");
assert.deepStrictEqual(flagged.findings.map((f) => f.source), ["ms-unslop:7", "ms-unslop:7"]);
const first = hook("prompt", { transcript_path: transcript }, { MOSAIC_UNSLOP_LOG: log });
assert.strictEqual(first.status, 0, first.stderr);
const out = JSON.parse(first.stdout).hookSpecificOutput;
assert.strictEqual(out.hookEventName, "UserPromptSubmit");
assert.ok(out.additionalContext.startsWith("UNSLOP NOTICE (mechanical style check, not the user speaking)"), out.additionalContext);
assert.ok(out.additionalContext.includes('banned word "delve" x1'), out.additionalContext);
const again = hook("prompt", { transcript_path: transcript }, { MOSAIC_UNSLOP_LOG: log });
assert.strictEqual(again.status, 0);
assert.strictEqual(again.stdout, "", "one notice per reply");
assert.deepStrictEqual(events(log).map((e) => e.ev), ["flagged", "notice_injected"]);
// Off: nothing logged, nothing printed.
const off = tmp("hook-off.jsonl");
const quiet = hook("prompt", { transcript_path: transcript }, { MOSAIC_UNSLOP_LOG: off, MOSAIC_UNSLOP_HOOK: "0" });
assert.strictEqual(quiet.status, 0);
assert.strictEqual(quiet.stdout, "");
assert.ok(!fs.existsSync(off));
});
t("claude-hook: a broken lists.json logs broken_lists and no verdict; bad input logs hook_failed; both exit 0", () => {
const transcript = writeTmp("transcript-clean.jsonl", JSON.stringify(line("assistant", "m1", "a1", [{ type: "text", text: "The build is green." }])) + "\n");
const log = tmp("hook-broken.jsonl");
for (const which of ["stop", "prompt"]) {
const r = hook(which, { transcript_path: transcript }, { MOSAIC_UNSLOP_LOG: log, UNSLOP_LISTS: writeTmp("hook-empty.json", "") });
assert.strictEqual(r.status, 0, r.stderr);
assert.strictEqual(r.stdout, "");
}
assert.deepStrictEqual(events(log).map((e) => [e.ev, e.at]), [["broken_lists", "stop"], ["broken_lists", "prompt"]]);
assert.ok(events(log).every((e) => /^lists\.json invalid: empty file/.test(e.reason)));
const bad = tmp("hook-bad.jsonl");
const r = hook("stop", "{not json", { MOSAIC_UNSLOP_LOG: bad });
assert.strictEqual(r.status, 0);
assert.deepStrictEqual(events(bad).map((e) => e.ev), ["hook_failed"]);
});
let failed = 0;
for (const [name, ok] of results) { console.log(`${ok ? "PASS" : "FAIL"} ${name}`); if (!ok) failed++; }
console.log(`${results.length - failed}/${results.length} passed`);
try { fs.rmSync(tmpdir, { recursive: true, force: true }); } catch {}
process.exit(failed ? 1 : 0);