diff --git a/packages/business/src/vars.mjs b/packages/business/src/vars.mjs index 9591ce528..69b6ea575 100644 --- a/packages/business/src/vars.mjs +++ b/packages/business/src/vars.mjs @@ -64,6 +64,11 @@ function list(check, { nonEmpty = true } = {}) { return (v, key) => requireDistinctList(v, `variable ${key}`, (item) => check(item, `${key} entry`), { nonEmpty }); } +function boolean(v, key) { + if (typeof v !== "boolean") refuse(`variable ${key} must be true or false (got ${JSON.stringify(v)})`); + return v; +} + function absolutePath(v, key) { if (typeof v !== "string" || !v.startsWith("/") || v.includes("\0")) refuse(`variable ${key} must be an absolute path`); return v; @@ -94,6 +99,8 @@ export const REGISTRY = Object.freeze({ // Project. "tracker.project": def(["project"], (v, key) => requirePositiveInt(v, `variable ${key}`)), + // On: the tracker project is the root of a tree, and sync reads its descendants too. + "tracker.descendants": def(["project"], boolean, { default: false }), "git.workingBranch": def(["project"], branch), "git.protectedBranches": def(["project"], list(branch)), suites: def(["project"], list(string(300))), diff --git a/packages/business/tests/vars.test.mjs b/packages/business/tests/vars.test.mjs index 251dbbd52..ab7d2b3cd 100644 --- a/packages/business/tests/vars.test.mjs +++ b/packages/business/tests/vars.test.mjs @@ -34,7 +34,7 @@ test("unknown keys and wrong layers refuse", () => { test("types", () => { const ok = (vars, layer) => assert.deepEqual({ ...checkVars(vars, layer, "v") }, vars); ok({ "tracker.baseUrl": "https://tasks.example:3456", "tracker.reconcileMinutes": 1, "tracker.pollSeconds": 10, "human.discordUserId": "123456789012345678" }, "business"); - ok({ "tracker.project": 3, "git.workingBranch": "refactor", "git.protectedBranches": ["main", "next"], suites: ["scripts/test-task.sh"], "issues.repo": "mosaicstack/stack" }, "project"); + ok({ "tracker.project": 3, "git.workingBranch": "refactor", "git.protectedBranches": ["main", "next"], suites: ["scripts/test-task.sh"], "issues.repo": "mosaicstack/stack", "tracker.descendants": true }, "project"); ok({ harness: "claude-code", model: "claude-opus-5-5", thinking: "high", "limits.tools": [], "limits.network": "none", "limits.authority": ["message.send"] }, "agent"); const bad = [ @@ -50,6 +50,9 @@ test("types", () => { [{ "human.discordUserId": 123456789012345678 }, "business", /as a string/], [{ "human.discordUserId": "0123456789012345678" }, "business", /Discord user id/], [{ "tracker.project": 0 }, "project", /positive integer/], + [{ "tracker.descendants": "true" }, "project", /true or false/], + [{ "tracker.descendants": 1 }, "project", /true or false/], + [{ "tracker.descendants": true }, "business", /business layer \(allowed: project\)/], [{ "git.workingBranch": "-x" }, "project", /git branch name/], [{ "git.workingBranch": "a..b" }, "project", /git branch name/], [{ "git.workingBranch": "x.lock" }, "project", /git branch name/], @@ -78,6 +81,8 @@ test("merge: defaults, then the most specific layer wins", () => { assert.equal(vars["tracker.reconcileMinutes"], 60); assert.equal(provenance["tracker.reconcileMinutes"], "default"); assert.equal(vars["tracker.kind"], "vikunja"); + assert.equal(vars["tracker.descendants"], false); + assert.equal(provenance["tracker.descendants"], "default"); assert.equal(vars.harness, undefined); assert.equal(provenance.harness, undefined); }); diff --git a/packages/cli/src/config.mjs b/packages/cli/src/config.mjs index d7315d6a1..e6373532c 100644 --- a/packages/cli/src/config.mjs +++ b/packages/cli/src/config.mjs @@ -78,6 +78,7 @@ function trackerFor(system, b, warn) { project: v["tracker.project"], pollSeconds: v["tracker.pollSeconds"], reconcileMinutes: v["tracker.reconcileMinutes"], + descendants: v["tracker.descendants"], }; } diff --git a/packages/cli/tests/config.test.mjs b/packages/cli/tests/config.test.mjs index cf384fc3d..b28cedaad 100644 --- a/packages/cli/tests/config.test.mjs +++ b/packages/cli/tests/config.test.mjs @@ -23,7 +23,11 @@ test("trackers come from the tracker.* variables of the one project that names a const f = fixture(root); writeJson(join(root, "project", ".mosaic", "project.json"), { projectVersion: 1, id: "stack", vars: { "tracker.project": 12 } }); const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env }); - assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60 } }); + assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60, descendants: false } }); + // Decision 87: the project's tracker.descendants reaches the adapter's boot entry. + writeJson(join(root, "project", ".mosaic", "project.json"), { projectVersion: 1, id: "stack", vars: { "tracker.project": 12, "tracker.descendants": true } }); + const tree = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env }); + assert.equal(tree.trackers.acme.descendants, true); }); test("with two projects, the one that sets tracker.project supplies the tracker and the other is no conflict", (t) => { @@ -35,7 +39,7 @@ test("with two projects, the one that sets tracker.project supplies the tracker writeJson(join(root, "project", ".mosaic", "project.json"), { projectVersion: 1, id: "stack", vars: { "tracker.project": 12 } }); writeJson(join(root, "web", ".mosaic", "project.json"), { projectVersion: 1, id: "web", vars: {} }); const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env }); - assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60 } }); + assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60, descendants: false } }); }); test("two projects that each name a tracker project refuse, since the boot shape holds one", (t) => { diff --git a/packages/cli/tests/trackers-boot.test.mjs b/packages/cli/tests/trackers-boot.test.mjs index b7710013e..d7cef73ab 100644 --- a/packages/cli/tests/trackers-boot.test.mjs +++ b/packages/cli/tests/trackers-boot.test.mjs @@ -41,7 +41,7 @@ test("bootConfig trackers reach the S3 adapter in the real broker child, which g mkdirSync(f.dataRoot, { recursive: true, mode: 0o700 }); const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env }); - assert.deepEqual(boot.trackers, { acme: { baseUrl: srv.url, project, pollSeconds: 60, reconcileMinutes: 60 } }); + assert.deepEqual(boot.trackers, { acme: { baseUrl: srv.url, project, pollSeconds: 60, reconcileMinutes: 60, descendants: false } }); const host = await startHost({ boot, business: "acme", log: () => {} }); t.after(() => host.close(0)); diff --git a/packages/tasks/README.md b/packages/tasks/README.md index 1b764e822..61b366647 100644 --- a/packages/tasks/README.md +++ b/packages/tasks/README.md @@ -1,7 +1,7 @@ # Tasks adapter, slice 1 S3 This package connects the broker to Vikunja v2. It handles the eight task -verbs, polls each business's project, and records what it reads as task +verbs, polls each business's project (or project tree), and records what it reads as task snapshots and events in `bus.sqlite`. It holds no token. Every call goes through the broker's `credentials.use()`, so the token goes into the Authorization header and nowhere else. No line it logs carries a token, @@ -20,7 +20,7 @@ is present: ```js {op: 'boot', config: {dataRoot, businesses, readers, repoRoots, trackers: {acme: {baseUrl: 'https://tasks.example.org', project: 12, - pollSeconds: 30, reconcileMinutes: 60}}}} + pollSeconds: 30, reconcileMinutes: 60, descendants: false}}}} ``` `baseUrl` is the origin only, for example `https://tasks.example.org`. The @@ -28,7 +28,8 @@ adapter adds `/api/v2` and refuses a URL with a path with `tracker-config`. S1's business validator accepts a path today, so a wrong value fails at boot, not at validation. That's a follow-up for S1. `pollSeconds` defaults to 30 and must be at least 10. `reconcileMinutes` -defaults to 60 and must be at least 1. +defaults to 60 and must be at least 1. `descendants` defaults to false +and must be a boolean. See "Project trees". From the business definition (`busBusiness` output) the adapter needs: @@ -88,7 +89,9 @@ until it passes every verb for that business refuses with sees a label only through its owner (probes.md, L), so `svc-$BIZ` owns the labels. -Then it runs one reconcile, and the business is `ready`. +Then it runs one reconcile, and the business is `ready`. With +`descendants` on, that reconcile also reads the project tree, so a 401 on +the project list refuses startup too. A startup refused with `tracker-unavailable`, `tracker-rate-limited` or `service-failed` is tried again at the next poll. Any other refusal @@ -239,6 +242,95 @@ mtime moved. `close()` stops the timers and waits for the queue. A verb still queued at that point refuses with `tracker-closed`. +## Project trees + +`tracker.descendants` is a project-layer business variable, false by +default. `packages/cli/src/config.mjs` passes it to the adapter as +`descendants` (lead decision 87). Off, the adapter reads one project, +exactly as before, and never reads the project list. + +On, the tracker project is the root of a tree. The sync token then also +needs `projects` `read_all`, for `GET /projects`. Without it startup +refuses with `tracker-unauthorized`. At startup and at each reconcile the +adapter reads that list, every page, and follows `parent_project_id` down +from the root to every depth. It skips negative ids (saved filters and +Favorites) and archived projects. Access in Vikunja inherits down the +tree, so a share on the root is enough for the sync bot to read every +child. The adapter writes no project, view or bucket, in the tree or out +of it. + +Startup checks the root's install as before. A root that fails refuses +the business with `tracker-install`. Each child gets the same install +checks at discovery: + +- A child that passes is installed. The adapter reads it through its + board, like the root. +- A child that fails is watch-only, and never refuses the business. The + adapter reads its open tasks from the task list and records them with + `bucket: 0` (`WATCH_ONLY`), because it has no board it trusts. Vikunja + bucket ids start at 1, so 0 is never a real bucket. Nothing maps 0 to + a bucket or a state. A move between a watch-only child's buckets + changes no recorded field. Closing or reopening the task still changes + `done`. +- A child whose views or buckets can't be read (5xx, no answer) is + neither. That refuses with `tracker-unavailable`, which is transient. + At startup the adapter tries again at the next poll. At a later + reconcile the reconcile fails, the tree from the last one stays, and + the next reconcile tries again. + +A task in a bucket outside the five, like a `Backlog` someone added, +keeps that bucket's raw id in its snapshot. + +Refs stay `vikunja:/`. Verbs act on the root only. A verb +on a child's task refuses with `task-project`, as it does for any other +project. + +A task moved between two projects of the tree gets `task.missing` with +reason `moved` and the new project under its old ref, and a first +snapshot under its new ref. A task moved out of the tree gets the same +tombstone and nothing after it. + +A project leaves the tree when it is archived, moved under a parent +outside the tree, no longer shared with the sync bot, or deleted. The +next reconcile drops it, or a tick before that (below), and each of its +tasks the bus holds is read once: + +- The sync bot can still read the task, in that project: `task.missing` + with `no-access`. The business no longer reaches it. +- The read answers 403: `no-access`. 404: `not-found`. +- The task moved to another project at the same time: `moved`. + +The tombstone keeps the task's last `updated`. Nothing in the ledger is +rewritten: the earlier snapshots stay and the tombstone is a new row. +A tick doesn't read the project list, so it keeps reading an archived +child, or one moved out with a share of its own, until that reconcile. +A child that answers 403 or 404 in a tick (no longer shared, moved under +a project the sync bot can't read, or deleted) leaves the tree at that +tick only if `GET /projects/` then answers 403 or 404 too, and its +tasks get the same one-by-one read. A deleted kanban view answers 404 +while its project still reads 200, so the child stays and the tick fails +until the next reconcile reads the child again, watch-only if it has no +installed kanban view. A 403 or 404 on the root +still fails the tick, and so does any other failure on a child. A new +child joins at the next reconcile, and so does a dropped one the sync +bot can read again. + +Each tick reads per project. An installed project costs the board and +the cursor walk, a watch-only one its open task list and the cursor +walk. That is two GETs per project, plus one per extra page of 50. A root +with three watch-only children, like the bus's #19, costs 8 GETs on a +quiet tick. On top of that a tick reads comments once per task the cursor +returned with a new `updated`, and reads each open task that left a +board once. A reconcile adds `GET /projects` (one page per 50 projects), +then each child's views and kanban buckets, then two GETs per project. +`tests/tree.test.mjs` counts both for a three-level tree. + +Vikunja v2.7.0 can also list tasks across every project the token reads +in one walk, `GET /api/v2/tasks` with a `project in ...` filter +(`agents/darkwing/work/queue-62/probes.md`). That would make a tick two +GETs whatever the tree's size. The adapter doesn't use it yet: it waits +on a live read with the sync key, and a switch is a follow-up row. + ## Refusals | Code | Meaning | @@ -277,6 +369,15 @@ every user or task any recording held, so a key one route adds, like The shape test catches a fake that invents a field, not one that puts a real field on the wrong route. +The fake also models project trees: `parent_project_id`, the project +list with archived projects and saved filters, and access inherited from +an ancestor's share (`project_access.go`: the nearest grant wins, admin +anywhere wins). No recording holds a `GET /projects` list, so +`shapes.test.mjs` checks its items against the recorded +`GET /projects/{id}` object, which has `parent_project_id` and +`is_archived`. +`tests/tree.test.mjs` covers `descendants`. + Nothing in these tests reaches a real Vikunja. The probes and the live run use a scratch container on 127.0.0.1, never a shared instance. diff --git a/packages/tasks/live/README.md b/packages/tasks/live/README.md index 186c4f2ea..478de618a 100644 --- a/packages/tasks/live/README.md +++ b/packages/tasks/live/README.md @@ -77,6 +77,12 @@ It exits 0 when every step went as expected, 1 when one didn't or the run stopped, and 2 on a bad command line or setup file. The task stays in the project, done, with "Safe to delete" in its description. +## Project trees + +The run reads the setup's one project, with `descendants` off. It never +reads a child. A tree is checked with read-only GETs instead, listed in +`agents/darkwing/work/queue-62/probes.md`. + ## The log `out/live-run.txt`, mode 0600, holds verbs, refusal codes, counts and diff --git a/packages/tasks/src/adapter.mjs b/packages/tasks/src/adapter.mjs index d564c6457..f0f8cacfe 100644 --- a/packages/tasks/src/adapter.mjs +++ b/packages/tasks/src/adapter.mjs @@ -5,7 +5,7 @@ import { startup } from './startup.mjs'; import { tick, reconcile } from './sync.mjs'; import { verbs } from './verbs.mjs'; // S3's adapter for startBroker({tasks}). `trackers` is the host's plain-data boot config: -// {: {baseUrl, project, pollSeconds, reconcileMinutes}}. A business with no entry has +// {: {baseUrl, project, pollSeconds, reconcileMinutes, descendants}}. A business with no entry has // no tracker, and its task verbs refuse. Startup runs in the background so the broker boots // without waiting on Vikunja; verbs refuse with tracker-starting until it passes. export const TIMEOUT = 60000; @@ -18,8 +18,10 @@ function settings(name, t, b) { apiRoot(t.baseUrl); const pollSeconds = t.pollSeconds ?? 30, reconcileMinutes = t.reconcileMinutes ?? 60; + const descendants = t.descendants ?? false; if (!positive(t.project) || !Number.isSafeInteger(pollSeconds) || pollSeconds < 10 || !positive(reconcileMinutes)) throw new BusError('tracker-config'); + if (typeof descendants !== 'boolean') throw new BusError('tracker-config'); if (!positive(b.tracker?.sync?.botId)) throw new BusError('tracker-config'); const roles = {}; for (const [role, r] of Object.entries(b.roles ?? {})) @@ -34,7 +36,7 @@ function settings(name, t, b) { if (!positive(id)) throw new BusError('tracker-config'); labels.set(id, title); } - return { business: name, baseUrl: t.baseUrl, project: t.project, pollSeconds, reconcileMinutes, roles, pm: pms[0], labels }; + return { business: name, baseUrl: t.baseUrl, project: t.project, pollSeconds, reconcileMinutes, descendants, roles, pm: pms[0], labels }; } // Files behind the business's Vikunja references, to notice a token rotated on disk. function files(b) { @@ -74,6 +76,7 @@ export function tasksAdapter({ call, per: 50, view: null, + tree: null, bots: new Set([businesses[name].tracker.sync.botId, ...Object.values(s.roles).map((r) => r.botId)]), sync: use('@sync'), as: (role, method, path, opts) => use(role)(method, path, opts), diff --git a/packages/tasks/src/fake.mjs b/packages/tasks/src/fake.mjs index fc55529a1..633224db8 100644 --- a/packages/tasks/src/fake.mjs +++ b/packages/tasks/src/fake.mjs @@ -6,9 +6,13 @@ import { NULL_DATE } from './digest.mjs'; // scope 401 before 404, bucket_id 0 outside the board and the move, `updated` reported to the // second but filtered on the stored value, moves between open buckets not bumping `updated`, // silent unassign, 403 on removing an absent label, and bot label visibility through the owner. +// Project trees follow the v2.7.0 source (row 62 probes.md): a grant on a project reaches every +// descendant, the nearest grant wins unless one on the path is admin, archiving a project archives +// its descendants, and the project list leaves out archived projects unless asked for them. const ROUTES = [ ['GET', /^\/info$/, null, 'info'], ['GET', /^\/labels$/, ['labels', 'read_all'], 'labels'], + ['GET', /^\/projects$/, ['projects', 'read_all'], 'projects'], ['GET', /^\/projects\/(\d+)$/, ['projects', 'read_one'], 'project'], ['GET', /^\/projects\/(\d+)\/views$/, ['projects_views', 'read_all'], 'views'], ['GET', /^\/projects\/(\d+)\/views\/(\d+)\/buckets$/, ['projects', 'views_buckets'], 'buckets'], @@ -80,6 +84,7 @@ export class FakeVikunja { #tasks = new Map(); #labels = new Map(); #faults = []; + #pseudo = []; constructor({ version = 'v2.7.0', now = () => Date.now() } = {}) { this.version = version; this.#now = now; @@ -98,9 +103,9 @@ export class FakeVikunja { revoke(token) { this.#tokens.delete(token); } - project(title, owner) { + project(title, owner, { parent = 0 } = {}) { const id = ++this.#seq.project; - this.#projects.set(id, { id, title, owner, shares: new Map(), index: 0 }); + this.#projects.set(id, { id, title, owner, parent, archived: !!this.#projects.get(parent)?.archived, shares: new Map(), index: 0 }); for (const [kind, position] of [ ['list', 100], ['gantt', 200], @@ -115,6 +120,34 @@ export class FakeVikunja { share(project, user, permission = 0) { this.#projects.get(project).shares.set(user, permission); } + unshare(project, user) { + this.#projects.get(project).shares.delete(user); + } + // Moves a project under another parent, or to the top level with 0. + reparent(project, parent) { + this.#projects.get(project).parent = parent; + } + // Vikunja writes is_archived down the tree, so a descendant reads as archived too. + archive(project, archived = true) { + for (const p of this.#projects.values()) if (p.id === project || this.#ancestors(p.id).includes(project)) p.archived = archived; + } + // Vikunja deletes a project's descendants and every task in them with it. + deleteProject(project) { + const gone = [...this.#projects.keys()].filter((id) => id === project || this.#ancestors(id).includes(project)); + for (const t of [...this.#tasks.values()]) if (gone.includes(t.project)) this.#tasks.delete(t.id); + for (const id of gone) this.#projects.delete(id); + } + // Drops a view, its buckets and the tasks' places in them. The project still reads, and a read + // of the view answers 404 with code 13001. + deleteView(view) { + for (const [id, b] of this.#buckets) if (b.view === view) this.#buckets.delete(id); + for (const t of this.#tasks.values()) t.buckets.delete(view); + this.#views.delete(view); + } + // A saved filter or Favorites: a negative id in the project list, as Project.ReadAll appends. + pseudoProject(user, id = -1, title = 'Favorites') { + this.#pseudo.push({ id, title, user }); + } bucket(view, title) { const id = ++this.#seq.bucket; this.#buckets.set(id, { id, title, view, position: id * 100 }); @@ -231,11 +264,39 @@ export class FakeVikunja { }); return id; } + // Parents, nearest first. A cycle can't be made through the setup methods the tests use. + #ancestors(project) { + const out = []; + for (let p = this.#projects.get(project)?.parent; p && !out.includes(p); p = this.#projects.get(p)?.parent) out.push(p); + return out; + } + // project_access.go: owner or a grant on the project or any ancestor; admin anywhere wins, + // otherwise the nearest grant. #access(user, project) { - const p = this.#projects.get(project); - if (!p) return null; - if (p.owner === user) return 2; - return p.shares.has(user) ? p.shares.get(user) : null; + if (!this.#projects.has(project)) return null; + let nearest = null; + for (const id of [project, ...this.#ancestors(project)]) { + const p = this.#projects.get(id); + if (!p) break; + if (p.owner === user) return 2; + const a = p.shares.has(user) ? p.shares.get(user) : null; + if (a === 2) return 2; + nearest ??= a; + } + return nearest; + } + #projectJson(p) { + return { + id: p.id, + title: p.title, + description: '', + owner: this.#userJson(p.owner), + parent_project_id: p.parent, + is_archived: p.archived, + position: p.id * 100, + created: second(0), + updated: second(0), + }; } #userJson(id) { const u = this.#users.get(id); @@ -378,7 +439,15 @@ export class FakeVikunja { _project(user, [p]) { const r = this.#projectFor(user, p); if (r.error) return r.error; - return { status: 200, body: { id: p, title: r.p.title, description: '', owner: this.#userJson(r.p.owner), is_archived: false } }; + return { status: 200, body: this.#projectJson(r.p) }; + } + _projects(user, _a, q) { + const archived = q.get('is_archived') === 'true'; + const list = [...this.#projects.values()] + .filter((p) => this.#access(user, p.id) !== null && (archived || !p.archived)) + .map((p) => this.#projectJson(p)); + for (const x of this.#pseudo) if (x.user === user) list.push({ id: x.id, title: x.title, description: '', owner: this.#userJson(user), parent_project_id: 0, is_archived: false }); + return { status: 200, body: page(list, q) }; } _views(user, [p], q) { const r = this.#projectFor(user, p); diff --git a/packages/tasks/src/startup.mjs b/packages/tasks/src/startup.mjs index 9a2c5bc59..f3bdc3d5d 100644 --- a/packages/tasks/src/startup.mjs +++ b/packages/tasks/src/startup.mjs @@ -35,7 +35,8 @@ export async function startup(ctx) { if (mine.some((s) => s.state === 'expired')) refuse('credential-expired'); // Sync identity: the control must pass and the write probe must be refused by scope. expect(await ctx.sync('GET', `/projects/${ctx.project}`), 200, 'tracker-project'); - await install(ctx); + ctx.view = await inspect(ctx, ctx.project); + ctx.tree = new Map([[ctx.project, ctx.view]]); const probe = await ctx.sync('PATCH', `/tasks/${MISSING}`, { body: {} }); if (probe.status === 404 || probe.status === 403 || outcome(probe) === 'ok') refuse('scope-too-broad'); expect(probe, 401, 'tracker-unexpected'); @@ -62,13 +63,14 @@ export async function startup(ctx) { } } // Install checks: one manual kanban view, the five buckets once each, done and default wired. -async function install(ctx) { - const views = await all(ctx.syncCall, null, `/projects/${ctx.project}/views`, {}, ctx.per); +// Returns the view, or refuses tracker-install. A failed read refuses tracker-unavailable. +async function inspect(ctx, project) { + const views = await all(ctx.syncCall, null, `/projects/${project}/views`, {}, ctx.per); if (!views.ok) refuse('tracker-unavailable'); const kanban = views.items.filter((x) => x?.view_kind === 'kanban'); if (kanban.length !== 1 || kanban[0].bucket_configuration_mode !== 'manual') refuse('tracker-install'); const k = kanban[0]; - const buckets = await all(ctx.syncCall, null, `/projects/${ctx.project}/views/${k.id}/buckets`, {}, ctx.per); + const buckets = await all(ctx.syncCall, null, `/projects/${project}/views/${k.id}/buckets`, {}, ctx.per); if (!buckets.ok) refuse('tracker-unavailable'); const ids = {}; for (const title of BUCKETS) { @@ -77,7 +79,7 @@ async function install(ctx) { ids[title] = hits[0].id; } if (k.done_bucket_id !== ids.done || k.default_bucket_id !== ids.todo) refuse('tracker-install'); - ctx.view = { + return { id: k.id, done: ids.done, todo: ids.todo, @@ -85,3 +87,32 @@ async function install(ctx) { titles: Object.fromEntries(Object.entries(ids).map(([t, id]) => [id, t])), }; } +// With tracker.descendants on, the tracker project is the root of a tree: the root plus every +// project the sync bot's project list shows under it by parent_project_id, archived ones left out. +// The first reconcile and each one after it run this. A child that passes the install checks is +// read through its board; one that fails is watch-only (null) and never refuses the business. +// ctx.tree maps each project to its view; the root keeps the view startup checked. +export async function discover(ctx) { + const r = await all(ctx.syncCall, null, '/projects', {}, ctx.per); + if (!r.ok) refuse(r.r.status === 401 ? 'tracker-unauthorized' : 'tracker-unavailable'); + const children = new Map(); + for (const p of r.items) { + if (!p || typeof p !== 'object' || !Number.isSafeInteger(p.id)) refuse('tracker-shape'); + // Saved filters and Favorites come back with negative ids. + if (p.id <= 0) continue; + if (!Number.isSafeInteger(p.parent_project_id) || p.parent_project_id < 0 || typeof p.is_archived !== 'boolean') refuse('tracker-shape'); + if (p.is_archived || p.id === ctx.project) continue; + if (!children.has(p.parent_project_id)) children.set(p.parent_project_id, []); + children.get(p.parent_project_id).push(p.id); + } + const tree = new Map([[ctx.project, ctx.view]]); + const next = [ctx.project]; + while (next.length) + for (const id of (children.get(next.shift()) ?? []).sort((a, b) => a - b)) { + if (tree.has(id)) continue; + tree.set(id, await inspect(ctx, id).catch((e) => (e instanceof BusError && e.code === 'tracker-install' ? null : Promise.reject(e)))); + next.push(id); + } + ctx.tree = tree; + return tree; +} diff --git a/packages/tasks/src/sync.mjs b/packages/tasks/src/sync.mjs index 811fb6834..852d2b2cf 100644 --- a/packages/tasks/src/sync.mjs +++ b/packages/tasks/src/sync.mjs @@ -1,10 +1,12 @@ import { BusError } from '../../bus/src/broker.mjs'; import { outcome, refusal, all } from './vikunja.mjs'; -import { ref, checkTask, taskFields, digest, tombstone } from './digest.mjs'; +import { ref, parseRef, checkTask, taskFields, digest, tombstone } from './digest.mjs'; +import { discover } from './startup.mjs'; // The sync bot's reads (addendum B sections 3 and 5). A tick reads the whole open board, because a // move between open buckets doesn't bump `updated`, then the cursor: every task updated since the // previous tick's start minus WINDOW_MS. The window covers the async bump lag and the one-second // rounding (probes.md, U and Paging). Overlap is deduped by digest against task_current. +// With tracker.descendants on, each tick and reconcile does this once per project in ctx.tree. export const WINDOW_MS = 60000; const iso = (ms) => new Date(ms).toISOString(); const floorSecond = (ms) => iso(Math.floor(ms / 1000) * 1000); @@ -14,11 +16,16 @@ export const normal = (updated) => iso(Date.parse(updated)); const shape = () => { throw new BusError('tracker-shape'); }; +// A watch-only project has no board the adapter trusts, so its snapshots carry bucket 0: no +// bucket state. Vikunja bucket ids start at 1, and nothing here reads 0 back as a bucket. +export const WATCH_ONLY = 0; +const fieldsFor = (task, bucket) => (bucket === WATCH_ONLY ? { ...taskFields(task, 1), bucket } : taskFields(task, bucket)); +const doneBucket = (view) => (view ? view.done : WATCH_ONLY); // The open board, every page. A bucket's tasks are null when it is empty; `count` is the bucket's total. -export async function board(ctx) { +export async function board(ctx, project = ctx.project, view = ctx.view) { const seen = new Map(); for (let page = 1; page <= 1000; page++) { - const r = await ctx.sync('GET', `/projects/${ctx.project}/views/${ctx.view.id}/buckets/tasks`, { + const r = await ctx.sync('GET', `/projects/${project}/views/${view.id}/buckets/tasks`, { query: { filter: 'done = false', per_page: ctx.per, page }, }); if (outcome(r) !== 'ok') throw new BusError(refusal(r)); @@ -37,12 +44,17 @@ export async function board(ctx) { } throw new BusError('tracker-paging'); } +// A watch-only project's open tasks, from its task list. +async function openTasks(ctx, project, view) { + if (view) return board(ctx, project, view); + return new Map((await walk(ctx, 'done = false', {}, project)).map((t) => [t.id, { task: t, bucket: WATCH_ONLY }])); +} // A list walk keyed by id, so a write landing mid-walk can't shift a page under it. -export async function walk(ctx, filter, query = {}) { +export async function walk(ctx, filter, query = {}, project = ctx.project) { const items = []; let last = 0; for (let i = 0; i < 100000; i++) { - const r = await ctx.sync('GET', `/projects/${ctx.project}/tasks`, { + const r = await ctx.sync('GET', `/projects/${project}/tasks`, { query: { ...query, filter: [filter, `id > ${last}`].filter(Boolean).join(' && '), @@ -56,7 +68,7 @@ export async function walk(ctx, filter, query = {}) { if (!Array.isArray(r.json?.items)) shape(); for (const t of r.json.items) { checkTask(t); - if (t.id <= last || t.project_id !== ctx.project) shape(); + if (t.id <= last || t.project_id !== project) shape(); last = t.id; items.push(t); } @@ -87,9 +99,9 @@ async function foreignComments(ctx, id, sinceMs) { } // One task read. Records nothing that a newer self snapshot already covers, and nothing that // matches task_current; a comment count alone is an event without a snapshot. -function consider(ctx, current, { task, bucket, via, readAt, comments = 0 }, out) { - const r = ref(ctx.project, task.id); - const fields = taskFields(task, bucket); +function consider(ctx, current, project, { task, bucket, via, readAt, comments = 0 }, out) { + const r = ref(project, task.id); + const fields = fieldsFor(task, bucket); const d = digest(fields); const c = current.get(r); const stale = c?.source === 'self' && readAt <= c.at; @@ -109,21 +121,27 @@ function consider(ctx, current, { task, bucket, via, readAt, comments = 0 }, out body: { via, digest: d, previous: c?.digest ?? null, changed: changed(c?.fields, fields), ...(comments ? { comments } : {}) }, }); } -// Open tasks the board no longer shows: done, moved, deleted or no longer shared. +// Open tasks the board no longer shows: done, moved, deleted or no longer shared. A task whose +// project left the tree (archived, moved out from under the root, or no longer shared) is out of +// the business's reach: no-access, or moved if the task itself changed project. async function missing(ctx, current, refs, out) { for (const r of refs) { const c = current.get(r); - const id = Number(r.slice(r.indexOf('/') + 1)); + const { project, task: id } = parseRef(r); const readAt = iso(ctx.clock()); const x = await ctx.sync('GET', `/tasks/${id}`); let fields, updated, event; if (x.status === 200) { checkTask(x.json); updated = normal(x.json.updated); - if (x.json.project_id !== ctx.project) { + if (x.json.project_id !== project) { fields = tombstone('moved', x.json.project_id); event = { reason: 'moved', project: x.json.project_id }; - } else if (x.json.done) fields = taskFields(x.json, ctx.view.done); + } else if (!ctx.tree.has(project)) { + updated = c.updated; + fields = tombstone('no-access'); + event = { reason: 'no-access' }; + } else if (x.json.done) fields = fieldsFor(x.json, doneBucket(ctx.tree.get(project))); else continue; // still open in the project: the board read raced a move, the next tick sees it } else if (outcome(x) === 'not-found' || x.status === 403) { updated = c.updated; @@ -147,39 +165,63 @@ async function missing(ctx, current, refs, out) { const record = (ctx, out) => { if (out.snapshots.length || out.events.length) ctx.broker.recordTask({ business: ctx.business, ...out }); }; +// A child that answers 403 or 404 may have left the tree after the last reconcile: no longer +// shared, moved under a project the bot can't read, or deleted. A 404 can also be the child's kanban +// view deleted while the project still reads, so the tick reads the project itself and drops the +// child only on a 403 or 404 there. Then missing() reads its open tasks one by one, as after a +// reconcile. Anything else fails the tick, and so does a 403 or 404 on the root. +async function leftTree(ctx, project, e) { + if (project === ctx.project || !(e instanceof BusError) || (e.code !== 'tracker-forbidden' && e.code !== 'tracker-not-found')) return false; + const r = await ctx.sync('GET', `/projects/${project}`); + return r.status === 403 || r.status === 404; +} export async function tick(ctx) { const startMs = ctx.clock(); const readAt = iso(startMs); const sinceMs = Math.floor(((ctx.lastStart ?? startMs) - WINDOW_MS) / 1000) * 1000; - const open = await board(ctx); - const hits = new Map((await walk(ctx, `updated > ${floorSecond(sinceMs)}`)).map((t) => [t.id, t])); + const reads = []; + for (const [project, view] of ctx.tree) { + try { + const open = await openTasks(ctx, project, view); + const hits = new Map((await walk(ctx, `updated > ${floorSecond(sinceMs)}`, {}, project)).map((t) => [t.id, t])); + reads.push({ project, view, open, hits }); + } catch (e) { + if (!(await leftTree(ctx, project, e))) throw e; + ctx.tree.delete(project); + } + } const current = new Map(ctx.broker.taskView(ctx.business, 'current').map((x) => [x.task_ref, x])); const out = { snapshots: [], events: [] }; - const comments = new Map(); - for (const [id, t] of hits) { - const key = normal(t.updated); - if (ctx.commentsAt.get(id) === key) continue; - comments.set(id, await foreignComments(ctx, id, sinceMs)); - ctx.commentsAt.set(id, key); - } - for (const [id, { task, bucket }] of open) { - const hit = hits.get(id); - const use = hit && Date.parse(hit.updated) >= Date.parse(task.updated) ? hit : task; - const b = use.done ? ctx.view.done : bucket; - consider(ctx, current, { task: use, bucket: b, via: hit ? 'cursor' : 'board', readAt, comments: comments.get(id) }, out); - } - const explained = new Set(); - for (const [id, t] of hits) { - if (open.has(id)) continue; - // An open task the board didn't show raced a move; the next board read places it. - if (!t.done) continue; - explained.add(ref(ctx.project, id)); - consider(ctx, current, { task: t, bucket: ctx.view.done, via: 'cursor', readAt, comments: comments.get(id) }, out); + // Refs some read this tick placed. Keyed by ref, not id: a task moved between two projects of the + // tree shows on the new one's board under the same id, and the old ref still needs its tombstone. + const placed = new Set(); + for (const { project, view, open, hits } of reads) { + const comments = new Map(); + for (const [id, t] of hits) { + const key = normal(t.updated); + if (ctx.commentsAt.get(id) === key) continue; + comments.set(id, await foreignComments(ctx, id, sinceMs)); + ctx.commentsAt.set(id, key); + } + for (const [id, { task, bucket }] of open) { + const hit = hits.get(id); + const use = hit && Date.parse(hit.updated) >= Date.parse(task.updated) ? hit : task; + const b = use.done ? doneBucket(view) : bucket; + placed.add(ref(project, id)); + consider(ctx, current, project, { task: use, bucket: b, via: hit ? 'cursor' : 'board', readAt, comments: comments.get(id) }, out); + } + for (const [id, t] of hits) { + if (open.has(id)) continue; + // An open task the board didn't show raced a move; the next board read places it. + if (!t.done) continue; + placed.add(ref(project, id)); + consider(ctx, current, project, { task: t, bucket: doneBucket(view), via: 'cursor', readAt, comments: comments.get(id) }, out); + } } const gone = ctx.broker .taskView(ctx.business, 'open') .map((x) => x.task_ref) - .filter((r) => !explained.has(r) && !open.has(Number(r.slice(r.indexOf('/') + 1)))); + .filter((r) => !placed.has(r)); await missing(ctx, current, gone, out); record(ctx, out); ctx.lastStart = startMs; @@ -187,27 +229,32 @@ export async function tick(ctx) { } // The hourly full read, and the first read at startup. comment_count catches comments on tasks the // cursor window missed; a changed count is checked against the comment list before it is reported. +// With tracker.descendants on it reads the project list first, so a new child joins the tree here +// and a child that left it has its tasks read one by one through missing(). export async function reconcile(ctx) { const startMs = ctx.clock(); const readAt = iso(startMs); const sinceMs = ctx.lastReconcile ?? startMs; - const open = await board(ctx); - const list = await walk(ctx, '', { expand: 'comment_count' }); + if (ctx.descendants) await discover(ctx); + const reads = []; + for (const [project, view] of ctx.tree) + reads.push({ project, view, open: await openTasks(ctx, project, view), list: await walk(ctx, '', { expand: 'comment_count' }, project) }); const current = new Map(ctx.broker.taskView(ctx.business, 'current').map((x) => [x.task_ref, x])); const out = { snapshots: [], events: [] }; const listed = new Set(); - for (const t of list) { - listed.add(ref(ctx.project, t.id)); - const bucket = t.done ? ctx.view.done : open.get(t.id)?.bucket; - if (!bucket) continue; - let comments = 0; - const count = t.comment_count; - if (count !== undefined && !Number.isSafeInteger(count)) shape(); - const before = ctx.commentCount.get(t.id); - if (count !== undefined) ctx.commentCount.set(t.id, count); - if (before !== undefined && count !== before) comments = await foreignComments(ctx, t.id, sinceMs); - consider(ctx, current, { task: t, bucket, via: 'reconcile', readAt, comments }, out); - } + for (const { project, view, open, list } of reads) + for (const t of list) { + listed.add(ref(project, t.id)); + const bucket = t.done ? doneBucket(view) : open.get(t.id)?.bucket; + if (bucket === undefined) continue; + let comments = 0; + const count = t.comment_count; + if (count !== undefined && !Number.isSafeInteger(count)) shape(); + const before = ctx.commentCount.get(t.id); + if (count !== undefined) ctx.commentCount.set(t.id, count); + if (before !== undefined && count !== before) comments = await foreignComments(ctx, t.id, sinceMs); + consider(ctx, current, project, { task: t, bucket, via: 'reconcile', readAt, comments }, out); + } const gone = [...current.values()] .filter((x) => x.fields.gone === undefined && !listed.has(x.task_ref)) .map((x) => x.task_ref); diff --git a/packages/tasks/tests/shapes.test.mjs b/packages/tasks/tests/shapes.test.mjs index 3859dd22a..c16f1f443 100644 --- a/packages/tasks/tests/shapes.test.mjs +++ b/packages/tasks/tests/shapes.test.mjs @@ -1,7 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; -import { world, sleep } from './world.mjs'; +import { world, sleep, SCOPES } from './world.mjs'; import { taskFields } from '../src/digest.mjs'; import { FakeVikunja } from '../src/fake.mjs'; const recorded = JSON.parse(readFileSync(new URL('./fixtures/v2-shapes.json', import.meta.url))); @@ -154,6 +154,30 @@ test('the fake answers each route with the statuses and shapes Vikunja v2.7.0 se for (const key of new Set(Object.values(KEY))) if (!UNPRODUCED.has(key)) assert.ok(covered.has(key), `the scenario never produced ${key}`); }); +// No recording holds a GET /projects list. Its envelope is the one every list sent, and each item +// is checked against the recorded GET /projects/{p} object, which has parent_project_id and is_archived. +test('the project list holds recorded project objects in the recorded list envelope', async (t) => { + const w = await world(t, { scopes: { sync: { ...SCOPES.sync, projects: [...SCOPES.sync.projects, 'read_all'] } } }); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + w.fake.pseudoProject(w.bots.sync); + const r = await w.fake.fetch('http://vikunja.test/api/v2/projects?per_page=50&page=1', { + headers: { authorization: 'Bearer ' + w.tokens.sync }, + }); + assert.equal(r.status, 200); + const body = await r.json(); + const problems = []; + compare({ ...body, items: [] }, samples['views 200'], 'projects', problems); + for (const p of body.items) compare(p, samples['project 200'], 'projects[]', problems); + assert.deepEqual(problems, []); + assert.deepEqual( + body.items.map((p) => [p.id, p.parent_project_id, p.is_archived]), + [ + [w.project, 0, false], + [child, w.project, false], + [-1, 0, false], + ], + ); +}); test('the recorded task bodies pass the checks S3 applies to every read', () => { const tasks = []; for (const [name, r] of Object.entries(recorded)) {