218 lines
12 KiB
Diff
218 lines
12 KiB
Diff
diff --git a/agents/rocko/work/queue-82/byte-check.sh b/agents/rocko/work/queue-82/byte-check.sh
|
|
new file mode 100755
|
|
index 00000000..4ec3c721
|
|
--- /dev/null
|
|
+++ b/agents/rocko/work/queue-82/byte-check.sh
|
|
@@ -0,0 +1,67 @@
|
|
+#!/usr/bin/env bash
|
|
+# Row 82 (#1551): compare `list` and `show` from two trees, byte for byte.
|
|
+# Usage: byte-check.sh <base-tree> <candidate-tree> <out-dir>.
|
|
+# Part 1 seeds a scratch data root with well-formed records only; part 2
|
|
+# reads the data root of the caller's own config (read-only: list and show
|
|
+# never write). Each command's stdout, stderr and exit status must match.
|
|
+# Part 3 seeds damaged records and only records both outputs, because the
|
|
+# base crashes there and the candidate shouldn't.
|
|
+set -euo pipefail
|
|
+BASE="$(cd "${1:?usage}" && pwd)"; CAND="$(cd "${2:?usage}" && pwd)"; OUT="${3:?usage}"
|
|
+mkdir -p "$OUT"
|
|
+S="$(mktemp -d)"; trap 'rm -rf "$S"' EXIT
|
|
+fail=0
|
|
+
|
|
+run() { # run TREE TAG CONFIG ARGS... -> $OUT/TAG.{out,err,rc}
|
|
+ local tree=$1 tag=$2 config=$3; shift 3
|
|
+ set +e
|
|
+ if [ -n "$config" ]; then MOSAIC_CONFIG="$config" node "$tree/scripts/mosaic-task.mjs" "$@" >"$OUT/$tag.out" 2>"$OUT/$tag.err"
|
|
+ else node "$tree/scripts/mosaic-task.mjs" "$@" >"$OUT/$tag.out" 2>"$OUT/$tag.err"; fi
|
|
+ echo $? >"$OUT/$tag.rc"
|
|
+ set -e
|
|
+}
|
|
+same() { # same NAME CONFIG ARGS...
|
|
+ local name=$1 config=$2; shift 2
|
|
+ run "$BASE" "$name.base" "$config" "$@"; run "$CAND" "$name.cand" "$config" "$@"
|
|
+ local x
|
|
+ for x in out err rc; do
|
|
+ if ! cmp -s "$OUT/$name.base.$x" "$OUT/$name.cand.$x"; then echo "DIFFERS $name ($x)"; fail=1; return; fi
|
|
+ done
|
|
+ echo "same $name (rc $(cat "$OUT/$name.base.rc"), $(wc -l <"$OUT/$name.base.out") lines)"
|
|
+}
|
|
+seed() { # seed DIR RUNID FILE JSON
|
|
+ mkdir -p "$1/runs/$2"; printf '%s\n' "$4" >"$1/runs/$2/$3"
|
|
+}
|
|
+config() { # config DATAROOT -> path of a scratch config for it
|
|
+ printf '{"configVersion":1,"environment":"development","dataRoot":"%s","execution":{"backend":"docker","provider":"zai","model":"m"}}\n' "$1" >"$1.config.json"
|
|
+ echo "$1.config.json"
|
|
+}
|
|
+
|
|
+# Part 1: well-formed records.
|
|
+D="$S/good"; mkdir -p "$D"
|
|
+R='"runVersion":1,"missionId":null,"reason":null,"request":"hi","response":"hi","expectedExact":null,"tools":null,"provider":"zai","model":"m","startedAt":"2026-01-01T00:00:00.000Z","finishedAt":"2026-01-01T00:00:01.000Z","durationMs":1000,"exitCode":0,"signal":null,"sessionForkFrom":null'
|
|
+seed "$D" r-20260101T000000Z-aaaaaa result.json "{$R,\"runId\":\"r-20260101T000000Z-aaaaaa\",\"taskId\":\"t-one\",\"status\":\"succeeded\",\"workspace\":\"ws1\",\"session\":\"s1\"}"
|
|
+seed "$D" r-20260101T000100Z-bbbbbb result.json "{$R,\"runId\":\"r-20260101T000100Z-bbbbbb\",\"taskId\":\"t-a-very-long-task-id-over-18\",\"status\":\"failed\",\"workspace\":null,\"session\":null}"
|
|
+seed "$D" r-20260101T000200Z-cccccc result.json "{$R,\"runId\":\"r-20260101T000200Z-cccccc\",\"taskId\":\"t-three\",\"status\":\"succeeded\",\"workspace\":\":run\",\"session\":null}"
|
|
+seed "$D" r-20260101T000300Z-dddddd result.json "{$R,\"runId\":\"r-20260101T000300Z-dddddd\",\"taskId\":\"t-four\",\"status\":\"timeout\"}"
|
|
+seed "$D" r-20260101T000400Z-eeeeee task.json '{"id":"t-five"}'
|
|
+seed "$D" r-.bad result.json "{$R,\"taskId\":\"t-bad\",\"status\":\"succeeded\"}"
|
|
+C="$(config "$D")"
|
|
+same good-list "$C" list
|
|
+for r in r-20260101T000000Z-aaaaaa r-20260101T000100Z-bbbbbb r-20260101T000200Z-cccccc r-20260101T000300Z-dddddd r-20260101T000400Z-eeeeee; do
|
|
+ same "good-show-$r" "$C" show "$r"
|
|
+done
|
|
+
|
|
+# Part 2: the caller's real data root, through its own config.
|
|
+same real-list "" list
|
|
+for r in $(awk '{print $1}' "$OUT/real-list.base.out"); do same "real-show-$r" "" show "$r"; done
|
|
+
|
|
+# Part 3: damaged records, recorded only.
|
|
+D="$S/damaged"; mkdir -p "$D"
|
|
+seed "$D" r-20260101T000000Z-aaaaaa result.json "{$R,\"taskId\":\"t-one\",\"status\":\"succeeded\",\"workspace\":\"ws1\",\"session\":\"s1\"}"
|
|
+seed "$D" r-20260101T000050Z-mmmmmm result.json '{}'
|
|
+seed "$D" r-20260101T000100Z-bbbbbb result.json "{$R,\"taskId\":\"t-two\",\"status\":\"succeeded\",\"workspace\":7,\"session\":{\"name\":\"s\"}}"
|
|
+C="$(config "$D")"
|
|
+run "$BASE" damaged-list.base "$C" list; run "$CAND" damaged-list.cand "$C" list
|
|
+echo "damaged list: base rc $(cat "$OUT/damaged-list.base.rc"), $(wc -l <"$OUT/damaged-list.base.out") lines; candidate rc $(cat "$OUT/damaged-list.cand.rc"), $(wc -l <"$OUT/damaged-list.cand.out") lines"
|
|
+exit "$fail"
|
|
diff --git a/agents/rocko/work/queue-82/mutants.sh b/agents/rocko/work/queue-82/mutants.sh
|
|
new file mode 100755
|
|
index 00000000..41ff04a8
|
|
--- /dev/null
|
|
+++ b/agents/rocko/work/queue-82/mutants.sh
|
|
@@ -0,0 +1,45 @@
|
|
+#!/usr/bin/env bash
|
|
+# Row 82 (#1551): rerun the listRuns mutants against a tree. Usage:
|
|
+# mutants.sh <tree>. The tree needs the candidate's listRuns; its tests can
|
|
+# be the candidate's or the base's. Each mutant drops one string check in
|
|
+# listRuns (scripts/mosaic-task.mjs), putting back that field's base line,
|
|
+# runs the runs suite, and restores the file from a saved copy, so the tree
|
|
+# need not be committed. Prints "killed" or "SURVIVED" per mutant; exit 1 if
|
|
+# any survives, a sed doesn't change the file, or the unmutated run fails.
|
|
+set -euo pipefail
|
|
+TREE="$(cd "${1:?usage: mutants.sh <tree>}" && pwd)"
|
|
+cd "$TREE"
|
|
+SRC=scripts/mosaic-task.mjs
|
|
+run_tests() { node --test 'packages/runs/tests/*.test.mjs' >"$1" 2>&1; }
|
|
+
|
|
+# name | sed expression
|
|
+MUTANTS=(
|
|
+ 'status-unchecked|s/^ if (typeof result\.status === "string") status = result\.status;$/ status = result.status;/'
|
|
+ 'taskid-unchecked|s/^ if (typeof result\.taskId === "string") taskId = result\.taskId;$/ taskId = result.taskId;/'
|
|
+ 'workspace-unchecked|s/^ if (typeof result\.workspace === "string") workspace = result\.workspace;$/ workspace = String(result.workspace ?? "-");/'
|
|
+ 'session-unchecked|s/^ if (typeof result\.session === "string") session = result\.session;$/ session = result.session ?? "-";/'
|
|
+)
|
|
+
|
|
+OUT="${OUT:-$TREE/.mutants}"
|
|
+mkdir -p "$OUT"
|
|
+run_tests "$OUT/baseline.log" || { echo "baseline run failed: $OUT/baseline.log" >&2; exit 1; }
|
|
+echo "baseline: pass"
|
|
+status=0
|
|
+for m in "${MUTANTS[@]}"; do
|
|
+ IFS='|' read -r name expr <<<"$m"
|
|
+ cp -p "$SRC" "$OUT/orig"
|
|
+ sed -i -e "$expr" "$SRC"
|
|
+ if cmp -s "$OUT/orig" "$SRC"; then
|
|
+ echo "$name: NOT APPLIED (sed matched nothing)"; status=1; continue
|
|
+ fi
|
|
+ diff -u "$OUT/orig" "$SRC" >"$OUT/$name.diff" || true
|
|
+ if run_tests "$OUT/$name.log"; then
|
|
+ echo "$name: SURVIVED"; status=1
|
|
+ else
|
|
+ echo "$name: killed ($(grep -E '^✖' "$OUT/$name.log" | grep -v 'failing tests' | sed 's/ ([0-9.]*ms)$//' | sort -u | tr '\n' ';'))"
|
|
+ fi
|
|
+ cp -p "$OUT/orig" "$SRC"
|
|
+ cmp -s "$OUT/orig" "$SRC" || { echo "restore failed: $SRC" >&2; exit 1; }
|
|
+ rm -f "$OUT/orig"
|
|
+done
|
|
+exit "$status"
|
|
diff --git a/packages/runs/README.md b/packages/runs/README.md
|
|
index 8fd78082..9718bed4 100644
|
|
--- a/packages/runs/README.md
|
|
+++ b/packages/runs/README.md
|
|
@@ -72,7 +72,10 @@ printed them.
|
|
|
|
`list` and `show` print exactly what they printed before this package
|
|
existed, for any data root with no link out of it, no run document that
|
|
-is JSON but not an object, and no unreadable directory.
|
|
+is JSON but not an object, no `result.json` with a field of the wrong
|
|
+type (a `status` or `taskId` that isn't a string, or a `workspace` or
|
|
+`session` that is neither a string nor `null`), and no unreadable
|
|
+directory.
|
|
`agents/rocko/work/queue-56/byte-check.sh` checks that byte for byte
|
|
against a seeded data root. Where those conditions don't hold, the
|
|
output changes on purpose:
|
|
@@ -82,6 +85,7 @@ output changes on purpose:
|
|
| run directory or `result.json` linked out | read through the link | `list`: `unknown`; `show`: refuses (run directory) or `result.json: (missing or unreadable)` |
|
|
| `runs/` linked out | read through the link | `list` and `show` refuse, exit 4 |
|
|
| `result.json` is `5` or `[]` | `list` crashed; `show` printed `undefined` fields | `unknown`; `(missing or unreadable)` |
|
|
+| `result.json` is an object whose `status` or `taskId` isn't a string, or whose `workspace` or `session` is neither a string nor `null` (`{}`, a numeric `status`, a `null` `taskId`) | `list` crashed on a `status` or `taskId` that isn't a string, hiding every run after it, and printed a `workspace` or `session` that is neither a string nor `null` through `String` | `list`: `unknown` for the status, `-` for each other field, and every later run still lists; `show` as before |
|
|
| `task.json` or `mission.json` is JSON but not an object | `show` printed its snapshot line | snapshot line omitted |
|
|
| run is a regular file | `show` crashed | `run not found`, exit 4 |
|
|
| `runs/` unreadable | `list` printed nothing | refuses, exit 4 |
|
|
diff --git a/packages/runs/tests/task-cli.test.mjs b/packages/runs/tests/task-cli.test.mjs
|
|
index ed13ecde..a7c1caed 100644
|
|
--- a/packages/runs/tests/task-cli.test.mjs
|
|
+++ b/packages/runs/tests/task-cli.test.mjs
|
|
@@ -32,6 +32,34 @@ test("list prints each run in the established format", (t) => {
|
|
`${B} unknown task=- ws=- session=-\n`);
|
|
});
|
|
|
|
+test("list prints a damaged result.json as unknown and keeps listing the runs after it", (t) => {
|
|
+ const { dataRoot } = scratch(t);
|
|
+ const M = "r-20260101T000050Z-mmmmmm";
|
|
+ write(path.join(dataRoot, "runs", A, "result.json"), { ...RESULT, workspace: "ws1", session: "s1" });
|
|
+ write(path.join(dataRoot, "runs", M, "result.json"), {});
|
|
+ write(path.join(dataRoot, "runs", B, "result.json"), { ...RESULT, taskId: "t-two", workspace: ":run" });
|
|
+ const out = task(t, dataRoot, "list");
|
|
+ assert.equal(out.status, 0, out.stderr);
|
|
+ assert.equal(out.stdout,
|
|
+ `${A} succeeded task=t-one ws=ws1 session=s1\n` +
|
|
+ `${M} unknown task=- ws=- session=-\n` +
|
|
+ `${B} succeeded task=t-two ws=:run session=-\n`);
|
|
+});
|
|
+
|
|
+test("list prints each result field that isn't a string as unknown or -", (t) => {
|
|
+ const { dataRoot } = scratch(t);
|
|
+ const runs = [
|
|
+ ["r-20260101T000000Z-status", { status: 5 }, "unknown task=t-one ws=- session=-"],
|
|
+ ["r-20260101T000001Z-taskid", { taskId: null }, "succeeded task=- ws=- session=-"],
|
|
+ ["r-20260101T000002Z-worksp", { workspace: 7 }, "succeeded task=t-one ws=- session=-"],
|
|
+ ["r-20260101T000003Z-sessin", { session: { name: "s1" } }, "succeeded task=t-one ws=- session=-"],
|
|
+ ];
|
|
+ for (const [runId, damage] of runs) write(path.join(dataRoot, "runs", runId, "result.json"), { ...RESULT, ...damage });
|
|
+ const out = task(t, dataRoot, "list");
|
|
+ assert.equal(out.status, 0, out.stderr);
|
|
+ assert.equal(out.stdout, runs.map(([runId, , line]) => `${runId} ${line}\n`).join(""));
|
|
+});
|
|
+
|
|
test("show prints the run in the established format", (t) => {
|
|
const { dataRoot } = scratch(t);
|
|
const dir = path.join(dataRoot, "runs", A);
|
|
diff --git a/scripts/mosaic-task.mjs b/scripts/mosaic-task.mjs
|
|
index cf8399a0..fc4e3845 100755
|
|
--- a/scripts/mosaic-task.mjs
|
|
+++ b/scripts/mosaic-task.mjs
|
|
@@ -470,18 +470,19 @@ function readRuns(read) {
|
|
function listRuns() {
|
|
const resolved = loadConfig();
|
|
for (const { runId, result } of readRuns(() => listRunRecords(resolved.dataRoot))) {
|
|
- // An incomplete or unreadable run record (result null) reports as unknown.
|
|
+ // An incomplete or unreadable run record (result null) reports as unknown,
|
|
+ // and so does each field of a damaged one that isn't a string.
|
|
let status = "unknown";
|
|
let taskId = "-";
|
|
let workspace = "-";
|
|
let session = "-";
|
|
if (result) {
|
|
- status = result.status;
|
|
- taskId = result.taskId;
|
|
- workspace = result.workspace ?? "-";
|
|
- session = result.session ?? "-";
|
|
+ if (typeof result.status === "string") status = result.status;
|
|
+ if (typeof result.taskId === "string") taskId = result.taskId;
|
|
+ if (typeof result.workspace === "string") workspace = result.workspace;
|
|
+ if (typeof result.session === "string") session = result.session;
|
|
}
|
|
- process.stdout.write(`${runId} ${status.padEnd(9)} task=${taskId.padEnd(18)} ws=${String(workspace).padEnd(10)} session=${session}\n`);
|
|
+ process.stdout.write(`${runId} ${status.padEnd(9)} task=${taskId.padEnd(18)} ws=${workspace.padEnd(10)} session=${session}\n`);
|
|
}
|
|
}
|
|
|