salvage 2026-11-06: uncommitted work from scratch/darkwing-scratch/r62/gate (disk cleanup #824)
This commit is contained in:
1 parent
ae428a5409
commit
dbfc201fdd
12 files changed
+371
-73
No files matched your search
@@ -64,6 +64,11 @@ function list(check, { nonEmpty = true } = {}) {
|
||||
return (v, key) => requireDistinctList(v, `variable ${key}`, (item) => check(item, `${key} entry`), { nonEmpty });
|
||||
}
|
||||
|
||||
function boolean(v, key) {
|
||||
if (typeof v !== "boolean") refuse(`variable ${key} must be true or false (got ${JSON.stringify(v)})`);
|
||||
return v;
|
||||
}
|
||||
|
||||
function absolutePath(v, key) {
|
||||
if (typeof v !== "string" || !v.startsWith("/") || v.includes("\0")) refuse(`variable ${key} must be an absolute path`);
|
||||
return v;
|
||||
@@ -94,6 +99,8 @@ export const REGISTRY = Object.freeze({
|
||||
|
||||
// Project.
|
||||
"tracker.project": def(["project"], (v, key) => requirePositiveInt(v, `variable ${key}`)),
|
||||
// On: the tracker project is the root of a tree, and sync reads its descendants too.
|
||||
"tracker.descendants": def(["project"], boolean, { default: false }),
|
||||
"git.workingBranch": def(["project"], branch),
|
||||
"git.protectedBranches": def(["project"], list(branch)),
|
||||
suites: def(["project"], list(string(300))),
|
||||
|
||||
@@ -34,7 +34,7 @@ test("unknown keys and wrong layers refuse", () => {
|
||||
test("types", () => {
|
||||
const ok = (vars, layer) => assert.deepEqual({ ...checkVars(vars, layer, "v") }, vars);
|
||||
ok({ "tracker.baseUrl": "https://tasks.example:3456", "tracker.reconcileMinutes": 1, "tracker.pollSeconds": 10, "human.discordUserId": "123456789012345678" }, "business");
|
||||
ok({ "tracker.project": 3, "git.workingBranch": "refactor", "git.protectedBranches": ["main", "next"], suites: ["scripts/test-task.sh"], "issues.repo": "mosaicstack/stack" }, "project");
|
||||
ok({ "tracker.project": 3, "git.workingBranch": "refactor", "git.protectedBranches": ["main", "next"], suites: ["scripts/test-task.sh"], "issues.repo": "mosaicstack/stack", "tracker.descendants": true }, "project");
|
||||
ok({ harness: "claude-code", model: "claude-opus-5-5", thinking: "high", "limits.tools": [], "limits.network": "none", "limits.authority": ["message.send"] }, "agent");
|
||||
|
||||
const bad = [
|
||||
@@ -50,6 +50,9 @@ test("types", () => {
|
||||
[{ "human.discordUserId": 123456789012345678 }, "business", /as a string/],
|
||||
[{ "human.discordUserId": "0123456789012345678" }, "business", /Discord user id/],
|
||||
[{ "tracker.project": 0 }, "project", /positive integer/],
|
||||
[{ "tracker.descendants": "true" }, "project", /true or false/],
|
||||
[{ "tracker.descendants": 1 }, "project", /true or false/],
|
||||
[{ "tracker.descendants": true }, "business", /business layer \(allowed: project\)/],
|
||||
[{ "git.workingBranch": "-x" }, "project", /git branch name/],
|
||||
[{ "git.workingBranch": "a..b" }, "project", /git branch name/],
|
||||
[{ "git.workingBranch": "x.lock" }, "project", /git branch name/],
|
||||
@@ -78,6 +81,8 @@ test("merge: defaults, then the most specific layer wins", () => {
|
||||
assert.equal(vars["tracker.reconcileMinutes"], 60);
|
||||
assert.equal(provenance["tracker.reconcileMinutes"], "default");
|
||||
assert.equal(vars["tracker.kind"], "vikunja");
|
||||
assert.equal(vars["tracker.descendants"], false);
|
||||
assert.equal(provenance["tracker.descendants"], "default");
|
||||
assert.equal(vars.harness, undefined);
|
||||
assert.equal(provenance.harness, undefined);
|
||||
});
|
||||
|
||||
@@ -78,6 +78,7 @@ function trackerFor(system, b, warn) {
|
||||
project: v["tracker.project"],
|
||||
pollSeconds: v["tracker.pollSeconds"],
|
||||
reconcileMinutes: v["tracker.reconcileMinutes"],
|
||||
descendants: v["tracker.descendants"],
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,11 @@ test("trackers come from the tracker.* variables of the one project that names a
|
||||
const f = fixture(root);
|
||||
writeJson(join(root, "project", ".mosaic", "project.json"), { projectVersion: 1, id: "stack", vars: { "tracker.project": 12 } });
|
||||
const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
|
||||
assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60 } });
|
||||
assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60, descendants: false } });
|
||||
// Decision 87: the project's tracker.descendants reaches the adapter's boot entry.
|
||||
writeJson(join(root, "project", ".mosaic", "project.json"), { projectVersion: 1, id: "stack", vars: { "tracker.project": 12, "tracker.descendants": true } });
|
||||
const tree = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
|
||||
assert.equal(tree.trackers.acme.descendants, true);
|
||||
});
|
||||
|
||||
test("with two projects, the one that sets tracker.project supplies the tracker and the other is no conflict", (t) => {
|
||||
@@ -35,7 +39,7 @@ test("with two projects, the one that sets tracker.project supplies the tracker
|
||||
writeJson(join(root, "project", ".mosaic", "project.json"), { projectVersion: 1, id: "stack", vars: { "tracker.project": 12 } });
|
||||
writeJson(join(root, "web", ".mosaic", "project.json"), { projectVersion: 1, id: "web", vars: {} });
|
||||
const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
|
||||
assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60 } });
|
||||
assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60, descendants: false } });
|
||||
});
|
||||
|
||||
test("two projects that each name a tracker project refuse, since the boot shape holds one", (t) => {
|
||||
|
||||
@@ -41,7 +41,7 @@ test("bootConfig trackers reach the S3 adapter in the real broker child, which g
|
||||
|
||||
mkdirSync(f.dataRoot, { recursive: true, mode: 0o700 });
|
||||
const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
|
||||
assert.deepEqual(boot.trackers, { acme: { baseUrl: srv.url, project, pollSeconds: 60, reconcileMinutes: 60 } });
|
||||
assert.deepEqual(boot.trackers, { acme: { baseUrl: srv.url, project, pollSeconds: 60, reconcileMinutes: 60, descendants: false } });
|
||||
const host = await startHost({ boot, business: "acme", log: () => {} });
|
||||
t.after(() => host.close(0));
|
||||
|
||||
|
||||
+105
-4
@@ -1,7 +1,7 @@
|
||||
# Tasks adapter, slice 1 S3
|
||||
|
||||
This package connects the broker to Vikunja v2. It handles the eight task
|
||||
verbs, polls each business's project, and records what it reads as task
|
||||
verbs, polls each business's project (or project tree), and records what it reads as task
|
||||
snapshots and events in `bus.sqlite`. It holds no token. Every call goes
|
||||
through the broker's `credentials.use()`, so the token goes into the
|
||||
Authorization header and nowhere else. No line it logs carries a token,
|
||||
@@ -20,7 +20,7 @@ is present:
|
||||
```js
|
||||
{op: 'boot', config: {dataRoot, businesses, readers, repoRoots,
|
||||
trackers: {acme: {baseUrl: 'https://tasks.example.org', project: 12,
|
||||
pollSeconds: 30, reconcileMinutes: 60}}}}
|
||||
pollSeconds: 30, reconcileMinutes: 60, descendants: false}}}}
|
||||
```
|
||||
|
||||
`baseUrl` is the origin only, for example `https://tasks.example.org`. The
|
||||
@@ -28,7 +28,8 @@ adapter adds `/api/v2` and refuses a URL with a path with
|
||||
`tracker-config`. S1's business validator accepts a path today, so a
|
||||
wrong value fails at boot, not at validation. That's a follow-up for S1.
|
||||
`pollSeconds` defaults to 30 and must be at least 10. `reconcileMinutes`
|
||||
defaults to 60 and must be at least 1.
|
||||
defaults to 60 and must be at least 1. `descendants` defaults to false
|
||||
and must be a boolean. See "Project trees".
|
||||
|
||||
From the business definition (`busBusiness` output) the adapter needs:
|
||||
|
||||
@@ -88,7 +89,9 @@ until it passes every verb for that business refuses with
|
||||
sees a label only through its owner (probes.md, L), so `svc-$BIZ`
|
||||
owns the labels.
|
||||
|
||||
Then it runs one reconcile, and the business is `ready`.
|
||||
Then it runs one reconcile, and the business is `ready`. With
|
||||
`descendants` on, that reconcile also reads the project tree, so a 401 on
|
||||
the project list refuses startup too.
|
||||
|
||||
A startup refused with `tracker-unavailable`, `tracker-rate-limited` or
|
||||
`service-failed` is tried again at the next poll. Any other refusal
|
||||
@@ -239,6 +242,95 @@ mtime moved.
|
||||
`close()` stops the timers and waits for the queue. A verb still queued
|
||||
at that point refuses with `tracker-closed`.
|
||||
|
||||
## Project trees
|
||||
|
||||
`tracker.descendants` is a project-layer business variable, false by
|
||||
default. `packages/cli/src/config.mjs` passes it to the adapter as
|
||||
`descendants` (lead decision 87). Off, the adapter reads one project,
|
||||
exactly as before, and never reads the project list.
|
||||
|
||||
On, the tracker project is the root of a tree. The sync token then also
|
||||
needs `projects` `read_all`, for `GET /projects`. Without it startup
|
||||
refuses with `tracker-unauthorized`. At startup and at each reconcile the
|
||||
adapter reads that list, every page, and follows `parent_project_id` down
|
||||
from the root to every depth. It skips negative ids (saved filters and
|
||||
Favorites) and archived projects. Access in Vikunja inherits down the
|
||||
tree, so a share on the root is enough for the sync bot to read every
|
||||
child. The adapter writes no project, view or bucket, in the tree or out
|
||||
of it.
|
||||
|
||||
Startup checks the root's install as before. A root that fails refuses
|
||||
the business with `tracker-install`. Each child gets the same install
|
||||
checks at discovery:
|
||||
|
||||
- A child that passes is installed. The adapter reads it through its
|
||||
board, like the root.
|
||||
- A child that fails is watch-only, and never refuses the business. The
|
||||
adapter reads its open tasks from the task list and records them with
|
||||
`bucket: 0` (`WATCH_ONLY`), because it has no board it trusts. Vikunja
|
||||
bucket ids start at 1, so 0 is never a real bucket. Nothing maps 0 to
|
||||
a bucket or a state. A move between a watch-only child's buckets
|
||||
changes no recorded field. Closing or reopening the task still changes
|
||||
`done`.
|
||||
- A child whose views or buckets can't be read (5xx, no answer) is
|
||||
neither. That refuses with `tracker-unavailable`, which is transient.
|
||||
At startup the adapter tries again at the next poll. At a later
|
||||
reconcile the reconcile fails, the tree from the last one stays, and
|
||||
the next reconcile tries again.
|
||||
|
||||
A task in a bucket outside the five, like a `Backlog` someone added,
|
||||
keeps that bucket's raw id in its snapshot.
|
||||
|
||||
Refs stay `vikunja:<project>/<task>`. Verbs act on the root only. A verb
|
||||
on a child's task refuses with `task-project`, as it does for any other
|
||||
project.
|
||||
|
||||
A task moved between two projects of the tree gets `task.missing` with
|
||||
reason `moved` and the new project under its old ref, and a first
|
||||
snapshot under its new ref. A task moved out of the tree gets the same
|
||||
tombstone and nothing after it.
|
||||
|
||||
A project leaves the tree when it is archived, moved under a parent
|
||||
outside the tree, no longer shared with the sync bot, or deleted. The
|
||||
next reconcile drops it, or a tick before that (below), and each of its
|
||||
tasks the bus holds is read once:
|
||||
|
||||
- The sync bot can still read the task, in that project: `task.missing`
|
||||
with `no-access`. The business no longer reaches it.
|
||||
- The read answers 403: `no-access`. 404: `not-found`.
|
||||
- The task moved to another project at the same time: `moved`.
|
||||
|
||||
The tombstone keeps the task's last `updated`. Nothing in the ledger is
|
||||
rewritten: the earlier snapshots stay and the tombstone is a new row.
|
||||
A tick doesn't read the project list, so it keeps reading an archived
|
||||
child, or one moved out with a share of its own, until that reconcile.
|
||||
A child that answers 403 or 404 in a tick (no longer shared, moved under
|
||||
a project the sync bot can't read, or deleted) leaves the tree at that
|
||||
tick only if `GET /projects/<id>` then answers 403 or 404 too, and its
|
||||
tasks get the same one-by-one read. A deleted kanban view answers 404
|
||||
while its project still reads 200, so the child stays and the tick fails
|
||||
until the next reconcile reads the child again, watch-only if it has no
|
||||
installed kanban view. A 403 or 404 on the root
|
||||
still fails the tick, and so does any other failure on a child. A new
|
||||
child joins at the next reconcile, and so does a dropped one the sync
|
||||
bot can read again.
|
||||
|
||||
Each tick reads per project. An installed project costs the board and
|
||||
the cursor walk, a watch-only one its open task list and the cursor
|
||||
walk. That is two GETs per project, plus one per extra page of 50. A root
|
||||
with three watch-only children, like the bus's #19, costs 8 GETs on a
|
||||
quiet tick. On top of that a tick reads comments once per task the cursor
|
||||
returned with a new `updated`, and reads each open task that left a
|
||||
board once. A reconcile adds `GET /projects` (one page per 50 projects),
|
||||
then each child's views and kanban buckets, then two GETs per project.
|
||||
`tests/tree.test.mjs` counts both for a three-level tree.
|
||||
|
||||
Vikunja v2.7.0 can also list tasks across every project the token reads
|
||||
in one walk, `GET /api/v2/tasks` with a `project in ...` filter
|
||||
(`agents/darkwing/work/queue-62/probes.md`). That would make a tick two
|
||||
GETs whatever the tree's size. The adapter doesn't use it yet: it waits
|
||||
on a live read with the sync key, and a switch is a follow-up row.
|
||||
|
||||
## Refusals
|
||||
|
||||
| Code | Meaning |
|
||||
@@ -277,6 +369,15 @@ every user or task any recording held, so a key one route adds, like
|
||||
The shape test catches a fake that invents a field, not one that puts a
|
||||
real field on the wrong route.
|
||||
|
||||
The fake also models project trees: `parent_project_id`, the project
|
||||
list with archived projects and saved filters, and access inherited from
|
||||
an ancestor's share (`project_access.go`: the nearest grant wins, admin
|
||||
anywhere wins). No recording holds a `GET /projects` list, so
|
||||
`shapes.test.mjs` checks its items against the recorded
|
||||
`GET /projects/{id}` object, which has `parent_project_id` and
|
||||
`is_archived`.
|
||||
`tests/tree.test.mjs` covers `descendants`.
|
||||
|
||||
Nothing in these tests reaches a real Vikunja. The probes and the live
|
||||
run use a scratch container on 127.0.0.1, never a shared instance.
|
||||
|
||||
|
||||
@@ -77,6 +77,12 @@ It exits 0 when every step went as expected, 1 when one didn't or the
|
||||
run stopped, and 2 on a bad command line or setup file. The task stays
|
||||
in the project, done, with "Safe to delete" in its description.
|
||||
|
||||
## Project trees
|
||||
|
||||
The run reads the setup's one project, with `descendants` off. It never
|
||||
reads a child. A tree is checked with read-only GETs instead, listed in
|
||||
`agents/darkwing/work/queue-62/probes.md`.
|
||||
|
||||
## The log
|
||||
|
||||
`out/live-run.txt`, mode 0600, holds verbs, refusal codes, counts and
|
||||
|
||||
@@ -5,7 +5,7 @@ import { startup } from './startup.mjs';
|
||||
import { tick, reconcile } from './sync.mjs';
|
||||
import { verbs } from './verbs.mjs';
|
||||
// S3's adapter for startBroker({tasks}). `trackers` is the host's plain-data boot config:
|
||||
// {<business>: {baseUrl, project, pollSeconds, reconcileMinutes}}. A business with no entry has
|
||||
// {<business>: {baseUrl, project, pollSeconds, reconcileMinutes, descendants}}. A business with no entry has
|
||||
// no tracker, and its task verbs refuse. Startup runs in the background so the broker boots
|
||||
// without waiting on Vikunja; verbs refuse with tracker-starting until it passes.
|
||||
export const TIMEOUT = 60000;
|
||||
@@ -18,8 +18,10 @@ function settings(name, t, b) {
|
||||
apiRoot(t.baseUrl);
|
||||
const pollSeconds = t.pollSeconds ?? 30,
|
||||
reconcileMinutes = t.reconcileMinutes ?? 60;
|
||||
const descendants = t.descendants ?? false;
|
||||
if (!positive(t.project) || !Number.isSafeInteger(pollSeconds) || pollSeconds < 10 || !positive(reconcileMinutes))
|
||||
throw new BusError('tracker-config');
|
||||
if (typeof descendants !== 'boolean') throw new BusError('tracker-config');
|
||||
if (!positive(b.tracker?.sync?.botId)) throw new BusError('tracker-config');
|
||||
const roles = {};
|
||||
for (const [role, r] of Object.entries(b.roles ?? {}))
|
||||
@@ -34,7 +36,7 @@ function settings(name, t, b) {
|
||||
if (!positive(id)) throw new BusError('tracker-config');
|
||||
labels.set(id, title);
|
||||
}
|
||||
return { business: name, baseUrl: t.baseUrl, project: t.project, pollSeconds, reconcileMinutes, roles, pm: pms[0], labels };
|
||||
return { business: name, baseUrl: t.baseUrl, project: t.project, pollSeconds, reconcileMinutes, descendants, roles, pm: pms[0], labels };
|
||||
}
|
||||
// Files behind the business's Vikunja references, to notice a token rotated on disk.
|
||||
function files(b) {
|
||||
@@ -74,6 +76,7 @@ export function tasksAdapter({
|
||||
call,
|
||||
per: 50,
|
||||
view: null,
|
||||
tree: null,
|
||||
bots: new Set([businesses[name].tracker.sync.botId, ...Object.values(s.roles).map((r) => r.botId)]),
|
||||
sync: use('@sync'),
|
||||
as: (role, method, path, opts) => use(role)(method, path, opts),
|
||||
|
||||
@@ -6,9 +6,13 @@ import { NULL_DATE } from './digest.mjs';
|
||||
// scope 401 before 404, bucket_id 0 outside the board and the move, `updated` reported to the
|
||||
// second but filtered on the stored value, moves between open buckets not bumping `updated`,
|
||||
// silent unassign, 403 on removing an absent label, and bot label visibility through the owner.
|
||||
// Project trees follow the v2.7.0 source (row 62 probes.md): a grant on a project reaches every
|
||||
// descendant, the nearest grant wins unless one on the path is admin, archiving a project archives
|
||||
// its descendants, and the project list leaves out archived projects unless asked for them.
|
||||
const ROUTES = [
|
||||
['GET', /^\/info$/, null, 'info'],
|
||||
['GET', /^\/labels$/, ['labels', 'read_all'], 'labels'],
|
||||
['GET', /^\/projects$/, ['projects', 'read_all'], 'projects'],
|
||||
['GET', /^\/projects\/(\d+)$/, ['projects', 'read_one'], 'project'],
|
||||
['GET', /^\/projects\/(\d+)\/views$/, ['projects_views', 'read_all'], 'views'],
|
||||
['GET', /^\/projects\/(\d+)\/views\/(\d+)\/buckets$/, ['projects', 'views_buckets'], 'buckets'],
|
||||
@@ -80,6 +84,7 @@ export class FakeVikunja {
|
||||
#tasks = new Map();
|
||||
#labels = new Map();
|
||||
#faults = [];
|
||||
#pseudo = [];
|
||||
constructor({ version = 'v2.7.0', now = () => Date.now() } = {}) {
|
||||
this.version = version;
|
||||
this.#now = now;
|
||||
@@ -98,9 +103,9 @@ export class FakeVikunja {
|
||||
revoke(token) {
|
||||
this.#tokens.delete(token);
|
||||
}
|
||||
project(title, owner) {
|
||||
project(title, owner, { parent = 0 } = {}) {
|
||||
const id = ++this.#seq.project;
|
||||
this.#projects.set(id, { id, title, owner, shares: new Map(), index: 0 });
|
||||
this.#projects.set(id, { id, title, owner, parent, archived: !!this.#projects.get(parent)?.archived, shares: new Map(), index: 0 });
|
||||
for (const [kind, position] of [
|
||||
['list', 100],
|
||||
['gantt', 200],
|
||||
@@ -115,6 +120,34 @@ export class FakeVikunja {
|
||||
share(project, user, permission = 0) {
|
||||
this.#projects.get(project).shares.set(user, permission);
|
||||
}
|
||||
unshare(project, user) {
|
||||
this.#projects.get(project).shares.delete(user);
|
||||
}
|
||||
// Moves a project under another parent, or to the top level with 0.
|
||||
reparent(project, parent) {
|
||||
this.#projects.get(project).parent = parent;
|
||||
}
|
||||
// Vikunja writes is_archived down the tree, so a descendant reads as archived too.
|
||||
archive(project, archived = true) {
|
||||
for (const p of this.#projects.values()) if (p.id === project || this.#ancestors(p.id).includes(project)) p.archived = archived;
|
||||
}
|
||||
// Vikunja deletes a project's descendants and every task in them with it.
|
||||
deleteProject(project) {
|
||||
const gone = [...this.#projects.keys()].filter((id) => id === project || this.#ancestors(id).includes(project));
|
||||
for (const t of [...this.#tasks.values()]) if (gone.includes(t.project)) this.#tasks.delete(t.id);
|
||||
for (const id of gone) this.#projects.delete(id);
|
||||
}
|
||||
// Drops a view, its buckets and the tasks' places in them. The project still reads, and a read
|
||||
// of the view answers 404 with code 13001.
|
||||
deleteView(view) {
|
||||
for (const [id, b] of this.#buckets) if (b.view === view) this.#buckets.delete(id);
|
||||
for (const t of this.#tasks.values()) t.buckets.delete(view);
|
||||
this.#views.delete(view);
|
||||
}
|
||||
// A saved filter or Favorites: a negative id in the project list, as Project.ReadAll appends.
|
||||
pseudoProject(user, id = -1, title = 'Favorites') {
|
||||
this.#pseudo.push({ id, title, user });
|
||||
}
|
||||
bucket(view, title) {
|
||||
const id = ++this.#seq.bucket;
|
||||
this.#buckets.set(id, { id, title, view, position: id * 100 });
|
||||
@@ -231,11 +264,39 @@ export class FakeVikunja {
|
||||
});
|
||||
return id;
|
||||
}
|
||||
// Parents, nearest first. A cycle can't be made through the setup methods the tests use.
|
||||
#ancestors(project) {
|
||||
const out = [];
|
||||
for (let p = this.#projects.get(project)?.parent; p && !out.includes(p); p = this.#projects.get(p)?.parent) out.push(p);
|
||||
return out;
|
||||
}
|
||||
// project_access.go: owner or a grant on the project or any ancestor; admin anywhere wins,
|
||||
// otherwise the nearest grant.
|
||||
#access(user, project) {
|
||||
const p = this.#projects.get(project);
|
||||
if (!p) return null;
|
||||
if (p.owner === user) return 2;
|
||||
return p.shares.has(user) ? p.shares.get(user) : null;
|
||||
if (!this.#projects.has(project)) return null;
|
||||
let nearest = null;
|
||||
for (const id of [project, ...this.#ancestors(project)]) {
|
||||
const p = this.#projects.get(id);
|
||||
if (!p) break;
|
||||
if (p.owner === user) return 2;
|
||||
const a = p.shares.has(user) ? p.shares.get(user) : null;
|
||||
if (a === 2) return 2;
|
||||
nearest ??= a;
|
||||
}
|
||||
return nearest;
|
||||
}
|
||||
#projectJson(p) {
|
||||
return {
|
||||
id: p.id,
|
||||
title: p.title,
|
||||
description: '',
|
||||
owner: this.#userJson(p.owner),
|
||||
parent_project_id: p.parent,
|
||||
is_archived: p.archived,
|
||||
position: p.id * 100,
|
||||
created: second(0),
|
||||
updated: second(0),
|
||||
};
|
||||
}
|
||||
#userJson(id) {
|
||||
const u = this.#users.get(id);
|
||||
@@ -378,7 +439,15 @@ export class FakeVikunja {
|
||||
_project(user, [p]) {
|
||||
const r = this.#projectFor(user, p);
|
||||
if (r.error) return r.error;
|
||||
return { status: 200, body: { id: p, title: r.p.title, description: '', owner: this.#userJson(r.p.owner), is_archived: false } };
|
||||
return { status: 200, body: this.#projectJson(r.p) };
|
||||
}
|
||||
_projects(user, _a, q) {
|
||||
const archived = q.get('is_archived') === 'true';
|
||||
const list = [...this.#projects.values()]
|
||||
.filter((p) => this.#access(user, p.id) !== null && (archived || !p.archived))
|
||||
.map((p) => this.#projectJson(p));
|
||||
for (const x of this.#pseudo) if (x.user === user) list.push({ id: x.id, title: x.title, description: '', owner: this.#userJson(user), parent_project_id: 0, is_archived: false });
|
||||
return { status: 200, body: page(list, q) };
|
||||
}
|
||||
_views(user, [p], q) {
|
||||
const r = this.#projectFor(user, p);
|
||||
|
||||
@@ -35,7 +35,8 @@ export async function startup(ctx) {
|
||||
if (mine.some((s) => s.state === 'expired')) refuse('credential-expired');
|
||||
// Sync identity: the control must pass and the write probe must be refused by scope.
|
||||
expect(await ctx.sync('GET', `/projects/${ctx.project}`), 200, 'tracker-project');
|
||||
await install(ctx);
|
||||
ctx.view = await inspect(ctx, ctx.project);
|
||||
ctx.tree = new Map([[ctx.project, ctx.view]]);
|
||||
const probe = await ctx.sync('PATCH', `/tasks/${MISSING}`, { body: {} });
|
||||
if (probe.status === 404 || probe.status === 403 || outcome(probe) === 'ok') refuse('scope-too-broad');
|
||||
expect(probe, 401, 'tracker-unexpected');
|
||||
@@ -62,13 +63,14 @@ export async function startup(ctx) {
|
||||
}
|
||||
}
|
||||
// Install checks: one manual kanban view, the five buckets once each, done and default wired.
|
||||
async function install(ctx) {
|
||||
const views = await all(ctx.syncCall, null, `/projects/${ctx.project}/views`, {}, ctx.per);
|
||||
// Returns the view, or refuses tracker-install. A failed read refuses tracker-unavailable.
|
||||
async function inspect(ctx, project) {
|
||||
const views = await all(ctx.syncCall, null, `/projects/${project}/views`, {}, ctx.per);
|
||||
if (!views.ok) refuse('tracker-unavailable');
|
||||
const kanban = views.items.filter((x) => x?.view_kind === 'kanban');
|
||||
if (kanban.length !== 1 || kanban[0].bucket_configuration_mode !== 'manual') refuse('tracker-install');
|
||||
const k = kanban[0];
|
||||
const buckets = await all(ctx.syncCall, null, `/projects/${ctx.project}/views/${k.id}/buckets`, {}, ctx.per);
|
||||
const buckets = await all(ctx.syncCall, null, `/projects/${project}/views/${k.id}/buckets`, {}, ctx.per);
|
||||
if (!buckets.ok) refuse('tracker-unavailable');
|
||||
const ids = {};
|
||||
for (const title of BUCKETS) {
|
||||
@@ -77,7 +79,7 @@ async function install(ctx) {
|
||||
ids[title] = hits[0].id;
|
||||
}
|
||||
if (k.done_bucket_id !== ids.done || k.default_bucket_id !== ids.todo) refuse('tracker-install');
|
||||
ctx.view = {
|
||||
return {
|
||||
id: k.id,
|
||||
done: ids.done,
|
||||
todo: ids.todo,
|
||||
@@ -85,3 +87,32 @@ async function install(ctx) {
|
||||
titles: Object.fromEntries(Object.entries(ids).map(([t, id]) => [id, t])),
|
||||
};
|
||||
}
|
||||
// With tracker.descendants on, the tracker project is the root of a tree: the root plus every
|
||||
// project the sync bot's project list shows under it by parent_project_id, archived ones left out.
|
||||
// The first reconcile and each one after it run this. A child that passes the install checks is
|
||||
// read through its board; one that fails is watch-only (null) and never refuses the business.
|
||||
// ctx.tree maps each project to its view; the root keeps the view startup checked.
|
||||
export async function discover(ctx) {
|
||||
const r = await all(ctx.syncCall, null, '/projects', {}, ctx.per);
|
||||
if (!r.ok) refuse(r.r.status === 401 ? 'tracker-unauthorized' : 'tracker-unavailable');
|
||||
const children = new Map();
|
||||
for (const p of r.items) {
|
||||
if (!p || typeof p !== 'object' || !Number.isSafeInteger(p.id)) refuse('tracker-shape');
|
||||
// Saved filters and Favorites come back with negative ids.
|
||||
if (p.id <= 0) continue;
|
||||
if (!Number.isSafeInteger(p.parent_project_id) || p.parent_project_id < 0 || typeof p.is_archived !== 'boolean') refuse('tracker-shape');
|
||||
if (p.is_archived || p.id === ctx.project) continue;
|
||||
if (!children.has(p.parent_project_id)) children.set(p.parent_project_id, []);
|
||||
children.get(p.parent_project_id).push(p.id);
|
||||
}
|
||||
const tree = new Map([[ctx.project, ctx.view]]);
|
||||
const next = [ctx.project];
|
||||
while (next.length)
|
||||
for (const id of (children.get(next.shift()) ?? []).sort((a, b) => a - b)) {
|
||||
if (tree.has(id)) continue;
|
||||
tree.set(id, await inspect(ctx, id).catch((e) => (e instanceof BusError && e.code === 'tracker-install' ? null : Promise.reject(e))));
|
||||
next.push(id);
|
||||
}
|
||||
ctx.tree = tree;
|
||||
return tree;
|
||||
}
|
||||
+97
-50
@@ -1,10 +1,12 @@
|
||||
import { BusError } from '../../bus/src/broker.mjs';
|
||||
import { outcome, refusal, all } from './vikunja.mjs';
|
||||
import { ref, checkTask, taskFields, digest, tombstone } from './digest.mjs';
|
||||
import { ref, parseRef, checkTask, taskFields, digest, tombstone } from './digest.mjs';
|
||||
import { discover } from './startup.mjs';
|
||||
// The sync bot's reads (addendum B sections 3 and 5). A tick reads the whole open board, because a
|
||||
// move between open buckets doesn't bump `updated`, then the cursor: every task updated since the
|
||||
// previous tick's start minus WINDOW_MS. The window covers the async bump lag and the one-second
|
||||
// rounding (probes.md, U and Paging). Overlap is deduped by digest against task_current.
|
||||
// With tracker.descendants on, each tick and reconcile does this once per project in ctx.tree.
|
||||
export const WINDOW_MS = 60000;
|
||||
const iso = (ms) => new Date(ms).toISOString();
|
||||
const floorSecond = (ms) => iso(Math.floor(ms / 1000) * 1000);
|
||||
@@ -14,11 +16,16 @@ export const normal = (updated) => iso(Date.parse(updated));
|
||||
const shape = () => {
|
||||
throw new BusError('tracker-shape');
|
||||
};
|
||||
// A watch-only project has no board the adapter trusts, so its snapshots carry bucket 0: no
|
||||
// bucket state. Vikunja bucket ids start at 1, and nothing here reads 0 back as a bucket.
|
||||
export const WATCH_ONLY = 0;
|
||||
const fieldsFor = (task, bucket) => (bucket === WATCH_ONLY ? { ...taskFields(task, 1), bucket } : taskFields(task, bucket));
|
||||
const doneBucket = (view) => (view ? view.done : WATCH_ONLY);
|
||||
// The open board, every page. A bucket's tasks are null when it is empty; `count` is the bucket's total.
|
||||
export async function board(ctx) {
|
||||
export async function board(ctx, project = ctx.project, view = ctx.view) {
|
||||
const seen = new Map();
|
||||
for (let page = 1; page <= 1000; page++) {
|
||||
const r = await ctx.sync('GET', `/projects/${ctx.project}/views/${ctx.view.id}/buckets/tasks`, {
|
||||
const r = await ctx.sync('GET', `/projects/${project}/views/${view.id}/buckets/tasks`, {
|
||||
query: { filter: 'done = false', per_page: ctx.per, page },
|
||||
});
|
||||
if (outcome(r) !== 'ok') throw new BusError(refusal(r));
|
||||
@@ -37,12 +44,17 @@ export async function board(ctx) {
|
||||
}
|
||||
throw new BusError('tracker-paging');
|
||||
}
|
||||
// A watch-only project's open tasks, from its task list.
|
||||
async function openTasks(ctx, project, view) {
|
||||
if (view) return board(ctx, project, view);
|
||||
return new Map((await walk(ctx, 'done = false', {}, project)).map((t) => [t.id, { task: t, bucket: WATCH_ONLY }]));
|
||||
}
|
||||
// A list walk keyed by id, so a write landing mid-walk can't shift a page under it.
|
||||
export async function walk(ctx, filter, query = {}) {
|
||||
export async function walk(ctx, filter, query = {}, project = ctx.project) {
|
||||
const items = [];
|
||||
let last = 0;
|
||||
for (let i = 0; i < 100000; i++) {
|
||||
const r = await ctx.sync('GET', `/projects/${ctx.project}/tasks`, {
|
||||
const r = await ctx.sync('GET', `/projects/${project}/tasks`, {
|
||||
query: {
|
||||
...query,
|
||||
filter: [filter, `id > ${last}`].filter(Boolean).join(' && '),
|
||||
@@ -56,7 +68,7 @@ export async function walk(ctx, filter, query = {}) {
|
||||
if (!Array.isArray(r.json?.items)) shape();
|
||||
for (const t of r.json.items) {
|
||||
checkTask(t);
|
||||
if (t.id <= last || t.project_id !== ctx.project) shape();
|
||||
if (t.id <= last || t.project_id !== project) shape();
|
||||
last = t.id;
|
||||
items.push(t);
|
||||
}
|
||||
@@ -87,9 +99,9 @@ async function foreignComments(ctx, id, sinceMs) {
|
||||
}
|
||||
// One task read. Records nothing that a newer self snapshot already covers, and nothing that
|
||||
// matches task_current; a comment count alone is an event without a snapshot.
|
||||
function consider(ctx, current, { task, bucket, via, readAt, comments = 0 }, out) {
|
||||
const r = ref(ctx.project, task.id);
|
||||
const fields = taskFields(task, bucket);
|
||||
function consider(ctx, current, project, { task, bucket, via, readAt, comments = 0 }, out) {
|
||||
const r = ref(project, task.id);
|
||||
const fields = fieldsFor(task, bucket);
|
||||
const d = digest(fields);
|
||||
const c = current.get(r);
|
||||
const stale = c?.source === 'self' && readAt <= c.at;
|
||||
@@ -109,21 +121,27 @@ function consider(ctx, current, { task, bucket, via, readAt, comments = 0 }, out
|
||||
body: { via, digest: d, previous: c?.digest ?? null, changed: changed(c?.fields, fields), ...(comments ? { comments } : {}) },
|
||||
});
|
||||
}
|
||||
// Open tasks the board no longer shows: done, moved, deleted or no longer shared.
|
||||
// Open tasks the board no longer shows: done, moved, deleted or no longer shared. A task whose
|
||||
// project left the tree (archived, moved out from under the root, or no longer shared) is out of
|
||||
// the business's reach: no-access, or moved if the task itself changed project.
|
||||
async function missing(ctx, current, refs, out) {
|
||||
for (const r of refs) {
|
||||
const c = current.get(r);
|
||||
const id = Number(r.slice(r.indexOf('/') + 1));
|
||||
const { project, task: id } = parseRef(r);
|
||||
const readAt = iso(ctx.clock());
|
||||
const x = await ctx.sync('GET', `/tasks/${id}`);
|
||||
let fields, updated, event;
|
||||
if (x.status === 200) {
|
||||
checkTask(x.json);
|
||||
updated = normal(x.json.updated);
|
||||
if (x.json.project_id !== ctx.project) {
|
||||
if (x.json.project_id !== project) {
|
||||
fields = tombstone('moved', x.json.project_id);
|
||||
event = { reason: 'moved', project: x.json.project_id };
|
||||
} else if (x.json.done) fields = taskFields(x.json, ctx.view.done);
|
||||
} else if (!ctx.tree.has(project)) {
|
||||
updated = c.updated;
|
||||
fields = tombstone('no-access');
|
||||
event = { reason: 'no-access' };
|
||||
} else if (x.json.done) fields = fieldsFor(x.json, doneBucket(ctx.tree.get(project)));
|
||||
else continue; // still open in the project: the board read raced a move, the next tick sees it
|
||||
} else if (outcome(x) === 'not-found' || x.status === 403) {
|
||||
updated = c.updated;
|
||||
@@ -147,39 +165,63 @@ async function missing(ctx, current, refs, out) {
|
||||
const record = (ctx, out) => {
|
||||
if (out.snapshots.length || out.events.length) ctx.broker.recordTask({ business: ctx.business, ...out });
|
||||
};
|
||||
// A child that answers 403 or 404 may have left the tree after the last reconcile: no longer
|
||||
// shared, moved under a project the bot can't read, or deleted. A 404 can also be the child's kanban
|
||||
// view deleted while the project still reads, so the tick reads the project itself and drops the
|
||||
// child only on a 403 or 404 there. Then missing() reads its open tasks one by one, as after a
|
||||
// reconcile. Anything else fails the tick, and so does a 403 or 404 on the root.
|
||||
async function leftTree(ctx, project, e) {
|
||||
if (project === ctx.project || !(e instanceof BusError) || (e.code !== 'tracker-forbidden' && e.code !== 'tracker-not-found')) return false;
|
||||
const r = await ctx.sync('GET', `/projects/${project}`);
|
||||
return r.status === 403 || r.status === 404;
|
||||
}
|
||||
export async function tick(ctx) {
|
||||
const startMs = ctx.clock();
|
||||
const readAt = iso(startMs);
|
||||
const sinceMs = Math.floor(((ctx.lastStart ?? startMs) - WINDOW_MS) / 1000) * 1000;
|
||||
const open = await board(ctx);
|
||||
const hits = new Map((await walk(ctx, `updated > ${floorSecond(sinceMs)}`)).map((t) => [t.id, t]));
|
||||
const reads = [];
|
||||
for (const [project, view] of ctx.tree) {
|
||||
try {
|
||||
const open = await openTasks(ctx, project, view);
|
||||
const hits = new Map((await walk(ctx, `updated > ${floorSecond(sinceMs)}`, {}, project)).map((t) => [t.id, t]));
|
||||
reads.push({ project, view, open, hits });
|
||||
} catch (e) {
|
||||
if (!(await leftTree(ctx, project, e))) throw e;
|
||||
ctx.tree.delete(project);
|
||||
}
|
||||
}
|
||||
const current = new Map(ctx.broker.taskView(ctx.business, 'current').map((x) => [x.task_ref, x]));
|
||||
const out = { snapshots: [], events: [] };
|
||||
const comments = new Map();
|
||||
for (const [id, t] of hits) {
|
||||
const key = normal(t.updated);
|
||||
if (ctx.commentsAt.get(id) === key) continue;
|
||||
comments.set(id, await foreignComments(ctx, id, sinceMs));
|
||||
ctx.commentsAt.set(id, key);
|
||||
}
|
||||
for (const [id, { task, bucket }] of open) {
|
||||
const hit = hits.get(id);
|
||||
const use = hit && Date.parse(hit.updated) >= Date.parse(task.updated) ? hit : task;
|
||||
const b = use.done ? ctx.view.done : bucket;
|
||||
consider(ctx, current, { task: use, bucket: b, via: hit ? 'cursor' : 'board', readAt, comments: comments.get(id) }, out);
|
||||
}
|
||||
const explained = new Set();
|
||||
for (const [id, t] of hits) {
|
||||
if (open.has(id)) continue;
|
||||
// An open task the board didn't show raced a move; the next board read places it.
|
||||
if (!t.done) continue;
|
||||
explained.add(ref(ctx.project, id));
|
||||
consider(ctx, current, { task: t, bucket: ctx.view.done, via: 'cursor', readAt, comments: comments.get(id) }, out);
|
||||
// Refs some read this tick placed. Keyed by ref, not id: a task moved between two projects of the
|
||||
// tree shows on the new one's board under the same id, and the old ref still needs its tombstone.
|
||||
const placed = new Set();
|
||||
for (const { project, view, open, hits } of reads) {
|
||||
const comments = new Map();
|
||||
for (const [id, t] of hits) {
|
||||
const key = normal(t.updated);
|
||||
if (ctx.commentsAt.get(id) === key) continue;
|
||||
comments.set(id, await foreignComments(ctx, id, sinceMs));
|
||||
ctx.commentsAt.set(id, key);
|
||||
}
|
||||
for (const [id, { task, bucket }] of open) {
|
||||
const hit = hits.get(id);
|
||||
const use = hit && Date.parse(hit.updated) >= Date.parse(task.updated) ? hit : task;
|
||||
const b = use.done ? doneBucket(view) : bucket;
|
||||
placed.add(ref(project, id));
|
||||
consider(ctx, current, project, { task: use, bucket: b, via: hit ? 'cursor' : 'board', readAt, comments: comments.get(id) }, out);
|
||||
}
|
||||
for (const [id, t] of hits) {
|
||||
if (open.has(id)) continue;
|
||||
// An open task the board didn't show raced a move; the next board read places it.
|
||||
if (!t.done) continue;
|
||||
placed.add(ref(project, id));
|
||||
consider(ctx, current, project, { task: t, bucket: doneBucket(view), via: 'cursor', readAt, comments: comments.get(id) }, out);
|
||||
}
|
||||
}
|
||||
const gone = ctx.broker
|
||||
.taskView(ctx.business, 'open')
|
||||
.map((x) => x.task_ref)
|
||||
.filter((r) => !explained.has(r) && !open.has(Number(r.slice(r.indexOf('/') + 1))));
|
||||
.filter((r) => !placed.has(r));
|
||||
await missing(ctx, current, gone, out);
|
||||
record(ctx, out);
|
||||
ctx.lastStart = startMs;
|
||||
@@ -187,27 +229,32 @@ export async function tick(ctx) {
|
||||
}
|
||||
// The hourly full read, and the first read at startup. comment_count catches comments on tasks the
|
||||
// cursor window missed; a changed count is checked against the comment list before it is reported.
|
||||
// With tracker.descendants on it reads the project list first, so a new child joins the tree here
|
||||
// and a child that left it has its tasks read one by one through missing().
|
||||
export async function reconcile(ctx) {
|
||||
const startMs = ctx.clock();
|
||||
const readAt = iso(startMs);
|
||||
const sinceMs = ctx.lastReconcile ?? startMs;
|
||||
const open = await board(ctx);
|
||||
const list = await walk(ctx, '', { expand: 'comment_count' });
|
||||
if (ctx.descendants) await discover(ctx);
|
||||
const reads = [];
|
||||
for (const [project, view] of ctx.tree)
|
||||
reads.push({ project, view, open: await openTasks(ctx, project, view), list: await walk(ctx, '', { expand: 'comment_count' }, project) });
|
||||
const current = new Map(ctx.broker.taskView(ctx.business, 'current').map((x) => [x.task_ref, x]));
|
||||
const out = { snapshots: [], events: [] };
|
||||
const listed = new Set();
|
||||
for (const t of list) {
|
||||
listed.add(ref(ctx.project, t.id));
|
||||
const bucket = t.done ? ctx.view.done : open.get(t.id)?.bucket;
|
||||
if (!bucket) continue;
|
||||
let comments = 0;
|
||||
const count = t.comment_count;
|
||||
if (count !== undefined && !Number.isSafeInteger(count)) shape();
|
||||
const before = ctx.commentCount.get(t.id);
|
||||
if (count !== undefined) ctx.commentCount.set(t.id, count);
|
||||
if (before !== undefined && count !== before) comments = await foreignComments(ctx, t.id, sinceMs);
|
||||
consider(ctx, current, { task: t, bucket, via: 'reconcile', readAt, comments }, out);
|
||||
}
|
||||
for (const { project, view, open, list } of reads)
|
||||
for (const t of list) {
|
||||
listed.add(ref(project, t.id));
|
||||
const bucket = t.done ? doneBucket(view) : open.get(t.id)?.bucket;
|
||||
if (bucket === undefined) continue;
|
||||
let comments = 0;
|
||||
const count = t.comment_count;
|
||||
if (count !== undefined && !Number.isSafeInteger(count)) shape();
|
||||
const before = ctx.commentCount.get(t.id);
|
||||
if (count !== undefined) ctx.commentCount.set(t.id, count);
|
||||
if (before !== undefined && count !== before) comments = await foreignComments(ctx, t.id, sinceMs);
|
||||
consider(ctx, current, project, { task: t, bucket, via: 'reconcile', readAt, comments }, out);
|
||||
}
|
||||
const gone = [...current.values()]
|
||||
.filter((x) => x.fields.gone === undefined && !listed.has(x.task_ref))
|
||||
.map((x) => x.task_ref);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { world, sleep } from './world.mjs';
|
||||
import { world, sleep, SCOPES } from './world.mjs';
|
||||
import { taskFields } from '../src/digest.mjs';
|
||||
import { FakeVikunja } from '../src/fake.mjs';
|
||||
const recorded = JSON.parse(readFileSync(new URL('./fixtures/v2-shapes.json', import.meta.url)));
|
||||
@@ -154,6 +154,30 @@ test('the fake answers each route with the statuses and shapes Vikunja v2.7.0 se
|
||||
for (const key of new Set(Object.values(KEY)))
|
||||
if (!UNPRODUCED.has(key)) assert.ok(covered.has(key), `the scenario never produced ${key}`);
|
||||
});
|
||||
// No recording holds a GET /projects list. Its envelope is the one every list sent, and each item
|
||||
// is checked against the recorded GET /projects/{p} object, which has parent_project_id and is_archived.
|
||||
test('the project list holds recorded project objects in the recorded list envelope', async (t) => {
|
||||
const w = await world(t, { scopes: { sync: { ...SCOPES.sync, projects: [...SCOPES.sync.projects, 'read_all'] } } });
|
||||
const child = w.fake.project('Child', w.owner, { parent: w.project });
|
||||
w.fake.pseudoProject(w.bots.sync);
|
||||
const r = await w.fake.fetch('http://vikunja.test/api/v2/projects?per_page=50&page=1', {
|
||||
headers: { authorization: 'Bearer ' + w.tokens.sync },
|
||||
});
|
||||
assert.equal(r.status, 200);
|
||||
const body = await r.json();
|
||||
const problems = [];
|
||||
compare({ ...body, items: [] }, samples['views 200'], 'projects', problems);
|
||||
for (const p of body.items) compare(p, samples['project 200'], 'projects[]', problems);
|
||||
assert.deepEqual(problems, []);
|
||||
assert.deepEqual(
|
||||
body.items.map((p) => [p.id, p.parent_project_id, p.is_archived]),
|
||||
[
|
||||
[w.project, 0, false],
|
||||
[child, w.project, false],
|
||||
[-1, 0, false],
|
||||
],
|
||||
);
|
||||
});
|
||||
test('the recorded task bodies pass the checks S3 applies to every read', () => {
|
||||
const tasks = [];
|
||||
for (const [name, r] of Object.entries(recorded)) {
|
||||
|
||||
Reference in new issue
Block a user