diff --git a/packages/business/src/vars.mjs b/packages/business/src/vars.mjs index 9591ce52..69b6ea57 100644 --- a/packages/business/src/vars.mjs +++ b/packages/business/src/vars.mjs @@ -64,6 +64,11 @@ function list(check, { nonEmpty = true } = {}) { return (v, key) => requireDistinctList(v, `variable ${key}`, (item) => check(item, `${key} entry`), { nonEmpty }); } +function boolean(v, key) { + if (typeof v !== "boolean") refuse(`variable ${key} must be true or false (got ${JSON.stringify(v)})`); + return v; +} + function absolutePath(v, key) { if (typeof v !== "string" || !v.startsWith("/") || v.includes("\0")) refuse(`variable ${key} must be an absolute path`); return v; @@ -94,6 +99,8 @@ export const REGISTRY = Object.freeze({ // Project. "tracker.project": def(["project"], (v, key) => requirePositiveInt(v, `variable ${key}`)), + // On: the tracker project is the root of a tree, and sync reads its descendants too. + "tracker.descendants": def(["project"], boolean, { default: false }), "git.workingBranch": def(["project"], branch), "git.protectedBranches": def(["project"], list(branch)), suites: def(["project"], list(string(300))), diff --git a/packages/business/tests/vars.test.mjs b/packages/business/tests/vars.test.mjs index 251dbbd5..ab7d2b3c 100644 --- a/packages/business/tests/vars.test.mjs +++ b/packages/business/tests/vars.test.mjs @@ -34,7 +34,7 @@ test("unknown keys and wrong layers refuse", () => { test("types", () => { const ok = (vars, layer) => assert.deepEqual({ ...checkVars(vars, layer, "v") }, vars); ok({ "tracker.baseUrl": "https://tasks.example:3456", "tracker.reconcileMinutes": 1, "tracker.pollSeconds": 10, "human.discordUserId": "123456789012345678" }, "business"); - ok({ "tracker.project": 3, "git.workingBranch": "refactor", "git.protectedBranches": ["main", "next"], suites: ["scripts/test-task.sh"], "issues.repo": "mosaicstack/stack" }, "project"); + ok({ "tracker.project": 3, "git.workingBranch": "refactor", "git.protectedBranches": ["main", "next"], suites: ["scripts/test-task.sh"], "issues.repo": "mosaicstack/stack", "tracker.descendants": true }, "project"); ok({ harness: "claude-code", model: "claude-opus-5-5", thinking: "high", "limits.tools": [], "limits.network": "none", "limits.authority": ["message.send"] }, "agent"); const bad = [ @@ -50,6 +50,9 @@ test("types", () => { [{ "human.discordUserId": 123456789012345678 }, "business", /as a string/], [{ "human.discordUserId": "0123456789012345678" }, "business", /Discord user id/], [{ "tracker.project": 0 }, "project", /positive integer/], + [{ "tracker.descendants": "true" }, "project", /true or false/], + [{ "tracker.descendants": 1 }, "project", /true or false/], + [{ "tracker.descendants": true }, "business", /business layer \(allowed: project\)/], [{ "git.workingBranch": "-x" }, "project", /git branch name/], [{ "git.workingBranch": "a..b" }, "project", /git branch name/], [{ "git.workingBranch": "x.lock" }, "project", /git branch name/], @@ -78,6 +81,8 @@ test("merge: defaults, then the most specific layer wins", () => { assert.equal(vars["tracker.reconcileMinutes"], 60); assert.equal(provenance["tracker.reconcileMinutes"], "default"); assert.equal(vars["tracker.kind"], "vikunja"); + assert.equal(vars["tracker.descendants"], false); + assert.equal(provenance["tracker.descendants"], "default"); assert.equal(vars.harness, undefined); assert.equal(provenance.harness, undefined); }); diff --git a/packages/cli/src/config.mjs b/packages/cli/src/config.mjs index d7315d6a..e6373532 100644 --- a/packages/cli/src/config.mjs +++ b/packages/cli/src/config.mjs @@ -78,6 +78,7 @@ function trackerFor(system, b, warn) { project: v["tracker.project"], pollSeconds: v["tracker.pollSeconds"], reconcileMinutes: v["tracker.reconcileMinutes"], + descendants: v["tracker.descendants"], }; } diff --git a/packages/cli/tests/config.test.mjs b/packages/cli/tests/config.test.mjs index cf384fc3..b28cedaa 100644 --- a/packages/cli/tests/config.test.mjs +++ b/packages/cli/tests/config.test.mjs @@ -23,7 +23,11 @@ test("trackers come from the tracker.* variables of the one project that names a const f = fixture(root); writeJson(join(root, "project", ".mosaic", "project.json"), { projectVersion: 1, id: "stack", vars: { "tracker.project": 12 } }); const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env }); - assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60 } }); + assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60, descendants: false } }); + // Decision 87: the project's tracker.descendants reaches the adapter's boot entry. + writeJson(join(root, "project", ".mosaic", "project.json"), { projectVersion: 1, id: "stack", vars: { "tracker.project": 12, "tracker.descendants": true } }); + const tree = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env }); + assert.equal(tree.trackers.acme.descendants, true); }); test("with two projects, the one that sets tracker.project supplies the tracker and the other is no conflict", (t) => { @@ -35,7 +39,7 @@ test("with two projects, the one that sets tracker.project supplies the tracker writeJson(join(root, "project", ".mosaic", "project.json"), { projectVersion: 1, id: "stack", vars: { "tracker.project": 12 } }); writeJson(join(root, "web", ".mosaic", "project.json"), { projectVersion: 1, id: "web", vars: {} }); const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env }); - assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60 } }); + assert.deepEqual(boot.trackers, { acme: { baseUrl: "http://127.0.0.1:3456", project: 12, pollSeconds: 60, reconcileMinutes: 60, descendants: false } }); }); test("two projects that each name a tracker project refuse, since the boot shape holds one", (t) => { diff --git a/packages/cli/tests/trackers-boot.test.mjs b/packages/cli/tests/trackers-boot.test.mjs index b7710013..d7cef73a 100644 --- a/packages/cli/tests/trackers-boot.test.mjs +++ b/packages/cli/tests/trackers-boot.test.mjs @@ -41,7 +41,7 @@ test("bootConfig trackers reach the S3 adapter in the real broker child, which g mkdirSync(f.dataRoot, { recursive: true, mode: 0o700 }); const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env }); - assert.deepEqual(boot.trackers, { acme: { baseUrl: srv.url, project, pollSeconds: 60, reconcileMinutes: 60 } }); + assert.deepEqual(boot.trackers, { acme: { baseUrl: srv.url, project, pollSeconds: 60, reconcileMinutes: 60, descendants: false } }); const host = await startHost({ boot, business: "acme", log: () => {} }); t.after(() => host.close(0)); diff --git a/packages/tasks/README.md b/packages/tasks/README.md index 1b764e82..61b36664 100644 --- a/packages/tasks/README.md +++ b/packages/tasks/README.md @@ -1,7 +1,7 @@ # Tasks adapter, slice 1 S3 This package connects the broker to Vikunja v2. It handles the eight task -verbs, polls each business's project, and records what it reads as task +verbs, polls each business's project (or project tree), and records what it reads as task snapshots and events in `bus.sqlite`. It holds no token. Every call goes through the broker's `credentials.use()`, so the token goes into the Authorization header and nowhere else. No line it logs carries a token, @@ -20,7 +20,7 @@ is present: ```js {op: 'boot', config: {dataRoot, businesses, readers, repoRoots, trackers: {acme: {baseUrl: 'https://tasks.example.org', project: 12, - pollSeconds: 30, reconcileMinutes: 60}}}} + pollSeconds: 30, reconcileMinutes: 60, descendants: false}}}} ``` `baseUrl` is the origin only, for example `https://tasks.example.org`. The @@ -28,7 +28,8 @@ adapter adds `/api/v2` and refuses a URL with a path with `tracker-config`. S1's business validator accepts a path today, so a wrong value fails at boot, not at validation. That's a follow-up for S1. `pollSeconds` defaults to 30 and must be at least 10. `reconcileMinutes` -defaults to 60 and must be at least 1. +defaults to 60 and must be at least 1. `descendants` defaults to false +and must be a boolean. See "Project trees". From the business definition (`busBusiness` output) the adapter needs: @@ -88,7 +89,9 @@ until it passes every verb for that business refuses with sees a label only through its owner (probes.md, L), so `svc-$BIZ` owns the labels. -Then it runs one reconcile, and the business is `ready`. +Then it runs one reconcile, and the business is `ready`. With +`descendants` on, that reconcile also reads the project tree, so a 401 on +the project list refuses startup too. A startup refused with `tracker-unavailable`, `tracker-rate-limited` or `service-failed` is tried again at the next poll. Any other refusal @@ -239,6 +242,95 @@ mtime moved. `close()` stops the timers and waits for the queue. A verb still queued at that point refuses with `tracker-closed`. +## Project trees + +`tracker.descendants` is a project-layer business variable, false by +default. `packages/cli/src/config.mjs` passes it to the adapter as +`descendants` (lead decision 87). Off, the adapter reads one project, +exactly as before, and never reads the project list. + +On, the tracker project is the root of a tree. The sync token then also +needs `projects` `read_all`, for `GET /projects`. Without it startup +refuses with `tracker-unauthorized`. At startup and at each reconcile the +adapter reads that list, every page, and follows `parent_project_id` down +from the root to every depth. It skips negative ids (saved filters and +Favorites) and archived projects. Access in Vikunja inherits down the +tree, so a share on the root is enough for the sync bot to read every +child. The adapter writes no project, view or bucket, in the tree or out +of it. + +Startup checks the root's install as before. A root that fails refuses +the business with `tracker-install`. Each child gets the same install +checks at discovery: + +- A child that passes is installed. The adapter reads it through its + board, like the root. +- A child that fails is watch-only, and never refuses the business. The + adapter reads its open tasks from the task list and records them with + `bucket: 0` (`WATCH_ONLY`), because it has no board it trusts. Vikunja + bucket ids start at 1, so 0 is never a real bucket. Nothing maps 0 to + a bucket or a state. A move between a watch-only child's buckets + changes no recorded field. Closing or reopening the task still changes + `done`. +- A child whose views or buckets can't be read (5xx, no answer) is + neither. That refuses with `tracker-unavailable`, which is transient. + At startup the adapter tries again at the next poll. At a later + reconcile the reconcile fails, the tree from the last one stays, and + the next reconcile tries again. + +A task in a bucket outside the five, like a `Backlog` someone added, +keeps that bucket's raw id in its snapshot. + +Refs stay `vikunja:/`. Verbs act on the root only. A verb +on a child's task refuses with `task-project`, as it does for any other +project. + +A task moved between two projects of the tree gets `task.missing` with +reason `moved` and the new project under its old ref, and a first +snapshot under its new ref. A task moved out of the tree gets the same +tombstone and nothing after it. + +A project leaves the tree when it is archived, moved under a parent +outside the tree, no longer shared with the sync bot, or deleted. The +next reconcile drops it, or a tick before that (below), and each of its +tasks the bus holds is read once: + +- The sync bot can still read the task, in that project: `task.missing` + with `no-access`. The business no longer reaches it. +- The read answers 403: `no-access`. 404: `not-found`. +- The task moved to another project at the same time: `moved`. + +The tombstone keeps the task's last `updated`. Nothing in the ledger is +rewritten: the earlier snapshots stay and the tombstone is a new row. +A tick doesn't read the project list, so it keeps reading an archived +child, or one moved out with a share of its own, until that reconcile. +A child that answers 403 or 404 in a tick (no longer shared, moved under +a project the sync bot can't read, or deleted) leaves the tree at that +tick only if `GET /projects/` then answers 403 or 404 too, and its +tasks get the same one-by-one read. A deleted kanban view answers 404 +while its project still reads 200, so the child stays and the tick fails +until the next reconcile reads the child again, watch-only if it has no +installed kanban view. A 403 or 404 on the root +still fails the tick, and so does any other failure on a child. A new +child joins at the next reconcile, and so does a dropped one the sync +bot can read again. + +Each tick reads per project. An installed project costs the board and +the cursor walk, a watch-only one its open task list and the cursor +walk. That is two GETs per project, plus one per extra page of 50. A root +with three watch-only children, like the bus's #19, costs 8 GETs on a +quiet tick. On top of that a tick reads comments once per task the cursor +returned with a new `updated`, and reads each open task that left a +board once. A reconcile adds `GET /projects` (one page per 50 projects), +then each child's views and kanban buckets, then two GETs per project. +`tests/tree.test.mjs` counts both for a three-level tree. + +Vikunja v2.7.0 can also list tasks across every project the token reads +in one walk, `GET /api/v2/tasks` with a `project in ...` filter +(`agents/darkwing/work/queue-62/probes.md`). That would make a tick two +GETs whatever the tree's size. The adapter doesn't use it yet: it waits +on a live read with the sync key, and a switch is a follow-up row. + ## Refusals | Code | Meaning | @@ -277,6 +369,15 @@ every user or task any recording held, so a key one route adds, like The shape test catches a fake that invents a field, not one that puts a real field on the wrong route. +The fake also models project trees: `parent_project_id`, the project +list with archived projects and saved filters, and access inherited from +an ancestor's share (`project_access.go`: the nearest grant wins, admin +anywhere wins). No recording holds a `GET /projects` list, so +`shapes.test.mjs` checks its items against the recorded +`GET /projects/{id}` object, which has `parent_project_id` and +`is_archived`. +`tests/tree.test.mjs` covers `descendants`. + Nothing in these tests reaches a real Vikunja. The probes and the live run use a scratch container on 127.0.0.1, never a shared instance. diff --git a/packages/tasks/live/README.md b/packages/tasks/live/README.md index 186c4f2e..478de618 100644 --- a/packages/tasks/live/README.md +++ b/packages/tasks/live/README.md @@ -77,6 +77,12 @@ It exits 0 when every step went as expected, 1 when one didn't or the run stopped, and 2 on a bad command line or setup file. The task stays in the project, done, with "Safe to delete" in its description. +## Project trees + +The run reads the setup's one project, with `descendants` off. It never +reads a child. A tree is checked with read-only GETs instead, listed in +`agents/darkwing/work/queue-62/probes.md`. + ## The log `out/live-run.txt`, mode 0600, holds verbs, refusal codes, counts and diff --git a/packages/tasks/src/adapter.mjs b/packages/tasks/src/adapter.mjs index d564c645..f0f8cacf 100644 --- a/packages/tasks/src/adapter.mjs +++ b/packages/tasks/src/adapter.mjs @@ -5,7 +5,7 @@ import { startup } from './startup.mjs'; import { tick, reconcile } from './sync.mjs'; import { verbs } from './verbs.mjs'; // S3's adapter for startBroker({tasks}). `trackers` is the host's plain-data boot config: -// {: {baseUrl, project, pollSeconds, reconcileMinutes}}. A business with no entry has +// {: {baseUrl, project, pollSeconds, reconcileMinutes, descendants}}. A business with no entry has // no tracker, and its task verbs refuse. Startup runs in the background so the broker boots // without waiting on Vikunja; verbs refuse with tracker-starting until it passes. export const TIMEOUT = 60000; @@ -18,8 +18,10 @@ function settings(name, t, b) { apiRoot(t.baseUrl); const pollSeconds = t.pollSeconds ?? 30, reconcileMinutes = t.reconcileMinutes ?? 60; + const descendants = t.descendants ?? false; if (!positive(t.project) || !Number.isSafeInteger(pollSeconds) || pollSeconds < 10 || !positive(reconcileMinutes)) throw new BusError('tracker-config'); + if (typeof descendants !== 'boolean') throw new BusError('tracker-config'); if (!positive(b.tracker?.sync?.botId)) throw new BusError('tracker-config'); const roles = {}; for (const [role, r] of Object.entries(b.roles ?? {})) @@ -34,7 +36,7 @@ function settings(name, t, b) { if (!positive(id)) throw new BusError('tracker-config'); labels.set(id, title); } - return { business: name, baseUrl: t.baseUrl, project: t.project, pollSeconds, reconcileMinutes, roles, pm: pms[0], labels }; + return { business: name, baseUrl: t.baseUrl, project: t.project, pollSeconds, reconcileMinutes, descendants, roles, pm: pms[0], labels }; } // Files behind the business's Vikunja references, to notice a token rotated on disk. function files(b) { @@ -74,6 +76,7 @@ export function tasksAdapter({ call, per: 50, view: null, + tree: null, bots: new Set([businesses[name].tracker.sync.botId, ...Object.values(s.roles).map((r) => r.botId)]), sync: use('@sync'), as: (role, method, path, opts) => use(role)(method, path, opts), diff --git a/packages/tasks/src/fake.mjs b/packages/tasks/src/fake.mjs index fc55529a..633224db 100644 --- a/packages/tasks/src/fake.mjs +++ b/packages/tasks/src/fake.mjs @@ -6,9 +6,13 @@ import { NULL_DATE } from './digest.mjs'; // scope 401 before 404, bucket_id 0 outside the board and the move, `updated` reported to the // second but filtered on the stored value, moves between open buckets not bumping `updated`, // silent unassign, 403 on removing an absent label, and bot label visibility through the owner. +// Project trees follow the v2.7.0 source (row 62 probes.md): a grant on a project reaches every +// descendant, the nearest grant wins unless one on the path is admin, archiving a project archives +// its descendants, and the project list leaves out archived projects unless asked for them. const ROUTES = [ ['GET', /^\/info$/, null, 'info'], ['GET', /^\/labels$/, ['labels', 'read_all'], 'labels'], + ['GET', /^\/projects$/, ['projects', 'read_all'], 'projects'], ['GET', /^\/projects\/(\d+)$/, ['projects', 'read_one'], 'project'], ['GET', /^\/projects\/(\d+)\/views$/, ['projects_views', 'read_all'], 'views'], ['GET', /^\/projects\/(\d+)\/views\/(\d+)\/buckets$/, ['projects', 'views_buckets'], 'buckets'], @@ -80,6 +84,7 @@ export class FakeVikunja { #tasks = new Map(); #labels = new Map(); #faults = []; + #pseudo = []; constructor({ version = 'v2.7.0', now = () => Date.now() } = {}) { this.version = version; this.#now = now; @@ -98,9 +103,9 @@ export class FakeVikunja { revoke(token) { this.#tokens.delete(token); } - project(title, owner) { + project(title, owner, { parent = 0 } = {}) { const id = ++this.#seq.project; - this.#projects.set(id, { id, title, owner, shares: new Map(), index: 0 }); + this.#projects.set(id, { id, title, owner, parent, archived: !!this.#projects.get(parent)?.archived, shares: new Map(), index: 0 }); for (const [kind, position] of [ ['list', 100], ['gantt', 200], @@ -115,6 +120,34 @@ export class FakeVikunja { share(project, user, permission = 0) { this.#projects.get(project).shares.set(user, permission); } + unshare(project, user) { + this.#projects.get(project).shares.delete(user); + } + // Moves a project under another parent, or to the top level with 0. + reparent(project, parent) { + this.#projects.get(project).parent = parent; + } + // Vikunja writes is_archived down the tree, so a descendant reads as archived too. + archive(project, archived = true) { + for (const p of this.#projects.values()) if (p.id === project || this.#ancestors(p.id).includes(project)) p.archived = archived; + } + // Vikunja deletes a project's descendants and every task in them with it. + deleteProject(project) { + const gone = [...this.#projects.keys()].filter((id) => id === project || this.#ancestors(id).includes(project)); + for (const t of [...this.#tasks.values()]) if (gone.includes(t.project)) this.#tasks.delete(t.id); + for (const id of gone) this.#projects.delete(id); + } + // Drops a view, its buckets and the tasks' places in them. The project still reads, and a read + // of the view answers 404 with code 13001. + deleteView(view) { + for (const [id, b] of this.#buckets) if (b.view === view) this.#buckets.delete(id); + for (const t of this.#tasks.values()) t.buckets.delete(view); + this.#views.delete(view); + } + // A saved filter or Favorites: a negative id in the project list, as Project.ReadAll appends. + pseudoProject(user, id = -1, title = 'Favorites') { + this.#pseudo.push({ id, title, user }); + } bucket(view, title) { const id = ++this.#seq.bucket; this.#buckets.set(id, { id, title, view, position: id * 100 }); @@ -231,11 +264,39 @@ export class FakeVikunja { }); return id; } + // Parents, nearest first. A cycle can't be made through the setup methods the tests use. + #ancestors(project) { + const out = []; + for (let p = this.#projects.get(project)?.parent; p && !out.includes(p); p = this.#projects.get(p)?.parent) out.push(p); + return out; + } + // project_access.go: owner or a grant on the project or any ancestor; admin anywhere wins, + // otherwise the nearest grant. #access(user, project) { - const p = this.#projects.get(project); - if (!p) return null; - if (p.owner === user) return 2; - return p.shares.has(user) ? p.shares.get(user) : null; + if (!this.#projects.has(project)) return null; + let nearest = null; + for (const id of [project, ...this.#ancestors(project)]) { + const p = this.#projects.get(id); + if (!p) break; + if (p.owner === user) return 2; + const a = p.shares.has(user) ? p.shares.get(user) : null; + if (a === 2) return 2; + nearest ??= a; + } + return nearest; + } + #projectJson(p) { + return { + id: p.id, + title: p.title, + description: '', + owner: this.#userJson(p.owner), + parent_project_id: p.parent, + is_archived: p.archived, + position: p.id * 100, + created: second(0), + updated: second(0), + }; } #userJson(id) { const u = this.#users.get(id); @@ -378,7 +439,15 @@ export class FakeVikunja { _project(user, [p]) { const r = this.#projectFor(user, p); if (r.error) return r.error; - return { status: 200, body: { id: p, title: r.p.title, description: '', owner: this.#userJson(r.p.owner), is_archived: false } }; + return { status: 200, body: this.#projectJson(r.p) }; + } + _projects(user, _a, q) { + const archived = q.get('is_archived') === 'true'; + const list = [...this.#projects.values()] + .filter((p) => this.#access(user, p.id) !== null && (archived || !p.archived)) + .map((p) => this.#projectJson(p)); + for (const x of this.#pseudo) if (x.user === user) list.push({ id: x.id, title: x.title, description: '', owner: this.#userJson(user), parent_project_id: 0, is_archived: false }); + return { status: 200, body: page(list, q) }; } _views(user, [p], q) { const r = this.#projectFor(user, p); diff --git a/packages/tasks/src/startup.mjs b/packages/tasks/src/startup.mjs index 9a2c5bc5..f3bdc3d5 100644 --- a/packages/tasks/src/startup.mjs +++ b/packages/tasks/src/startup.mjs @@ -35,7 +35,8 @@ export async function startup(ctx) { if (mine.some((s) => s.state === 'expired')) refuse('credential-expired'); // Sync identity: the control must pass and the write probe must be refused by scope. expect(await ctx.sync('GET', `/projects/${ctx.project}`), 200, 'tracker-project'); - await install(ctx); + ctx.view = await inspect(ctx, ctx.project); + ctx.tree = new Map([[ctx.project, ctx.view]]); const probe = await ctx.sync('PATCH', `/tasks/${MISSING}`, { body: {} }); if (probe.status === 404 || probe.status === 403 || outcome(probe) === 'ok') refuse('scope-too-broad'); expect(probe, 401, 'tracker-unexpected'); @@ -62,13 +63,14 @@ export async function startup(ctx) { } } // Install checks: one manual kanban view, the five buckets once each, done and default wired. -async function install(ctx) { - const views = await all(ctx.syncCall, null, `/projects/${ctx.project}/views`, {}, ctx.per); +// Returns the view, or refuses tracker-install. A failed read refuses tracker-unavailable. +async function inspect(ctx, project) { + const views = await all(ctx.syncCall, null, `/projects/${project}/views`, {}, ctx.per); if (!views.ok) refuse('tracker-unavailable'); const kanban = views.items.filter((x) => x?.view_kind === 'kanban'); if (kanban.length !== 1 || kanban[0].bucket_configuration_mode !== 'manual') refuse('tracker-install'); const k = kanban[0]; - const buckets = await all(ctx.syncCall, null, `/projects/${ctx.project}/views/${k.id}/buckets`, {}, ctx.per); + const buckets = await all(ctx.syncCall, null, `/projects/${project}/views/${k.id}/buckets`, {}, ctx.per); if (!buckets.ok) refuse('tracker-unavailable'); const ids = {}; for (const title of BUCKETS) { @@ -77,7 +79,7 @@ async function install(ctx) { ids[title] = hits[0].id; } if (k.done_bucket_id !== ids.done || k.default_bucket_id !== ids.todo) refuse('tracker-install'); - ctx.view = { + return { id: k.id, done: ids.done, todo: ids.todo, @@ -85,3 +87,32 @@ async function install(ctx) { titles: Object.fromEntries(Object.entries(ids).map(([t, id]) => [id, t])), }; } +// With tracker.descendants on, the tracker project is the root of a tree: the root plus every +// project the sync bot's project list shows under it by parent_project_id, archived ones left out. +// The first reconcile and each one after it run this. A child that passes the install checks is +// read through its board; one that fails is watch-only (null) and never refuses the business. +// ctx.tree maps each project to its view; the root keeps the view startup checked. +export async function discover(ctx) { + const r = await all(ctx.syncCall, null, '/projects', {}, ctx.per); + if (!r.ok) refuse(r.r.status === 401 ? 'tracker-unauthorized' : 'tracker-unavailable'); + const children = new Map(); + for (const p of r.items) { + if (!p || typeof p !== 'object' || !Number.isSafeInteger(p.id)) refuse('tracker-shape'); + // Saved filters and Favorites come back with negative ids. + if (p.id <= 0) continue; + if (!Number.isSafeInteger(p.parent_project_id) || p.parent_project_id < 0 || typeof p.is_archived !== 'boolean') refuse('tracker-shape'); + if (p.is_archived || p.id === ctx.project) continue; + if (!children.has(p.parent_project_id)) children.set(p.parent_project_id, []); + children.get(p.parent_project_id).push(p.id); + } + const tree = new Map([[ctx.project, ctx.view]]); + const next = [ctx.project]; + while (next.length) + for (const id of (children.get(next.shift()) ?? []).sort((a, b) => a - b)) { + if (tree.has(id)) continue; + tree.set(id, await inspect(ctx, id).catch((e) => (e instanceof BusError && e.code === 'tracker-install' ? null : Promise.reject(e)))); + next.push(id); + } + ctx.tree = tree; + return tree; +} diff --git a/packages/tasks/src/sync.mjs b/packages/tasks/src/sync.mjs index 811fb683..852d2b2c 100644 --- a/packages/tasks/src/sync.mjs +++ b/packages/tasks/src/sync.mjs @@ -1,10 +1,12 @@ import { BusError } from '../../bus/src/broker.mjs'; import { outcome, refusal, all } from './vikunja.mjs'; -import { ref, checkTask, taskFields, digest, tombstone } from './digest.mjs'; +import { ref, parseRef, checkTask, taskFields, digest, tombstone } from './digest.mjs'; +import { discover } from './startup.mjs'; // The sync bot's reads (addendum B sections 3 and 5). A tick reads the whole open board, because a // move between open buckets doesn't bump `updated`, then the cursor: every task updated since the // previous tick's start minus WINDOW_MS. The window covers the async bump lag and the one-second // rounding (probes.md, U and Paging). Overlap is deduped by digest against task_current. +// With tracker.descendants on, each tick and reconcile does this once per project in ctx.tree. export const WINDOW_MS = 60000; const iso = (ms) => new Date(ms).toISOString(); const floorSecond = (ms) => iso(Math.floor(ms / 1000) * 1000); @@ -14,11 +16,16 @@ export const normal = (updated) => iso(Date.parse(updated)); const shape = () => { throw new BusError('tracker-shape'); }; +// A watch-only project has no board the adapter trusts, so its snapshots carry bucket 0: no +// bucket state. Vikunja bucket ids start at 1, and nothing here reads 0 back as a bucket. +export const WATCH_ONLY = 0; +const fieldsFor = (task, bucket) => (bucket === WATCH_ONLY ? { ...taskFields(task, 1), bucket } : taskFields(task, bucket)); +const doneBucket = (view) => (view ? view.done : WATCH_ONLY); // The open board, every page. A bucket's tasks are null when it is empty; `count` is the bucket's total. -export async function board(ctx) { +export async function board(ctx, project = ctx.project, view = ctx.view) { const seen = new Map(); for (let page = 1; page <= 1000; page++) { - const r = await ctx.sync('GET', `/projects/${ctx.project}/views/${ctx.view.id}/buckets/tasks`, { + const r = await ctx.sync('GET', `/projects/${project}/views/${view.id}/buckets/tasks`, { query: { filter: 'done = false', per_page: ctx.per, page }, }); if (outcome(r) !== 'ok') throw new BusError(refusal(r)); @@ -37,12 +44,17 @@ export async function board(ctx) { } throw new BusError('tracker-paging'); } +// A watch-only project's open tasks, from its task list. +async function openTasks(ctx, project, view) { + if (view) return board(ctx, project, view); + return new Map((await walk(ctx, 'done = false', {}, project)).map((t) => [t.id, { task: t, bucket: WATCH_ONLY }])); +} // A list walk keyed by id, so a write landing mid-walk can't shift a page under it. -export async function walk(ctx, filter, query = {}) { +export async function walk(ctx, filter, query = {}, project = ctx.project) { const items = []; let last = 0; for (let i = 0; i < 100000; i++) { - const r = await ctx.sync('GET', `/projects/${ctx.project}/tasks`, { + const r = await ctx.sync('GET', `/projects/${project}/tasks`, { query: { ...query, filter: [filter, `id > ${last}`].filter(Boolean).join(' && '), @@ -56,7 +68,7 @@ export async function walk(ctx, filter, query = {}) { if (!Array.isArray(r.json?.items)) shape(); for (const t of r.json.items) { checkTask(t); - if (t.id <= last || t.project_id !== ctx.project) shape(); + if (t.id <= last || t.project_id !== project) shape(); last = t.id; items.push(t); } @@ -87,9 +99,9 @@ async function foreignComments(ctx, id, sinceMs) { } // One task read. Records nothing that a newer self snapshot already covers, and nothing that // matches task_current; a comment count alone is an event without a snapshot. -function consider(ctx, current, { task, bucket, via, readAt, comments = 0 }, out) { - const r = ref(ctx.project, task.id); - const fields = taskFields(task, bucket); +function consider(ctx, current, project, { task, bucket, via, readAt, comments = 0 }, out) { + const r = ref(project, task.id); + const fields = fieldsFor(task, bucket); const d = digest(fields); const c = current.get(r); const stale = c?.source === 'self' && readAt <= c.at; @@ -109,21 +121,27 @@ function consider(ctx, current, { task, bucket, via, readAt, comments = 0 }, out body: { via, digest: d, previous: c?.digest ?? null, changed: changed(c?.fields, fields), ...(comments ? { comments } : {}) }, }); } -// Open tasks the board no longer shows: done, moved, deleted or no longer shared. +// Open tasks the board no longer shows: done, moved, deleted or no longer shared. A task whose +// project left the tree (archived, moved out from under the root, or no longer shared) is out of +// the business's reach: no-access, or moved if the task itself changed project. async function missing(ctx, current, refs, out) { for (const r of refs) { const c = current.get(r); - const id = Number(r.slice(r.indexOf('/') + 1)); + const { project, task: id } = parseRef(r); const readAt = iso(ctx.clock()); const x = await ctx.sync('GET', `/tasks/${id}`); let fields, updated, event; if (x.status === 200) { checkTask(x.json); updated = normal(x.json.updated); - if (x.json.project_id !== ctx.project) { + if (x.json.project_id !== project) { fields = tombstone('moved', x.json.project_id); event = { reason: 'moved', project: x.json.project_id }; - } else if (x.json.done) fields = taskFields(x.json, ctx.view.done); + } else if (!ctx.tree.has(project)) { + updated = c.updated; + fields = tombstone('no-access'); + event = { reason: 'no-access' }; + } else if (x.json.done) fields = fieldsFor(x.json, doneBucket(ctx.tree.get(project))); else continue; // still open in the project: the board read raced a move, the next tick sees it } else if (outcome(x) === 'not-found' || x.status === 403) { updated = c.updated; @@ -147,39 +165,63 @@ async function missing(ctx, current, refs, out) { const record = (ctx, out) => { if (out.snapshots.length || out.events.length) ctx.broker.recordTask({ business: ctx.business, ...out }); }; +// A child that answers 403 or 404 may have left the tree after the last reconcile: no longer +// shared, moved under a project the bot can't read, or deleted. A 404 can also be the child's kanban +// view deleted while the project still reads, so the tick reads the project itself and drops the +// child only on a 403 or 404 there. Then missing() reads its open tasks one by one, as after a +// reconcile. Anything else fails the tick, and so does a 403 or 404 on the root. +async function leftTree(ctx, project, e) { + if (project === ctx.project || !(e instanceof BusError) || (e.code !== 'tracker-forbidden' && e.code !== 'tracker-not-found')) return false; + const r = await ctx.sync('GET', `/projects/${project}`); + return r.status === 403 || r.status === 404; +} export async function tick(ctx) { const startMs = ctx.clock(); const readAt = iso(startMs); const sinceMs = Math.floor(((ctx.lastStart ?? startMs) - WINDOW_MS) / 1000) * 1000; - const open = await board(ctx); - const hits = new Map((await walk(ctx, `updated > ${floorSecond(sinceMs)}`)).map((t) => [t.id, t])); + const reads = []; + for (const [project, view] of ctx.tree) { + try { + const open = await openTasks(ctx, project, view); + const hits = new Map((await walk(ctx, `updated > ${floorSecond(sinceMs)}`, {}, project)).map((t) => [t.id, t])); + reads.push({ project, view, open, hits }); + } catch (e) { + if (!(await leftTree(ctx, project, e))) throw e; + ctx.tree.delete(project); + } + } const current = new Map(ctx.broker.taskView(ctx.business, 'current').map((x) => [x.task_ref, x])); const out = { snapshots: [], events: [] }; - const comments = new Map(); - for (const [id, t] of hits) { - const key = normal(t.updated); - if (ctx.commentsAt.get(id) === key) continue; - comments.set(id, await foreignComments(ctx, id, sinceMs)); - ctx.commentsAt.set(id, key); - } - for (const [id, { task, bucket }] of open) { - const hit = hits.get(id); - const use = hit && Date.parse(hit.updated) >= Date.parse(task.updated) ? hit : task; - const b = use.done ? ctx.view.done : bucket; - consider(ctx, current, { task: use, bucket: b, via: hit ? 'cursor' : 'board', readAt, comments: comments.get(id) }, out); - } - const explained = new Set(); - for (const [id, t] of hits) { - if (open.has(id)) continue; - // An open task the board didn't show raced a move; the next board read places it. - if (!t.done) continue; - explained.add(ref(ctx.project, id)); - consider(ctx, current, { task: t, bucket: ctx.view.done, via: 'cursor', readAt, comments: comments.get(id) }, out); + // Refs some read this tick placed. Keyed by ref, not id: a task moved between two projects of the + // tree shows on the new one's board under the same id, and the old ref still needs its tombstone. + const placed = new Set(); + for (const { project, view, open, hits } of reads) { + const comments = new Map(); + for (const [id, t] of hits) { + const key = normal(t.updated); + if (ctx.commentsAt.get(id) === key) continue; + comments.set(id, await foreignComments(ctx, id, sinceMs)); + ctx.commentsAt.set(id, key); + } + for (const [id, { task, bucket }] of open) { + const hit = hits.get(id); + const use = hit && Date.parse(hit.updated) >= Date.parse(task.updated) ? hit : task; + const b = use.done ? doneBucket(view) : bucket; + placed.add(ref(project, id)); + consider(ctx, current, project, { task: use, bucket: b, via: hit ? 'cursor' : 'board', readAt, comments: comments.get(id) }, out); + } + for (const [id, t] of hits) { + if (open.has(id)) continue; + // An open task the board didn't show raced a move; the next board read places it. + if (!t.done) continue; + placed.add(ref(project, id)); + consider(ctx, current, project, { task: t, bucket: doneBucket(view), via: 'cursor', readAt, comments: comments.get(id) }, out); + } } const gone = ctx.broker .taskView(ctx.business, 'open') .map((x) => x.task_ref) - .filter((r) => !explained.has(r) && !open.has(Number(r.slice(r.indexOf('/') + 1)))); + .filter((r) => !placed.has(r)); await missing(ctx, current, gone, out); record(ctx, out); ctx.lastStart = startMs; @@ -187,27 +229,32 @@ export async function tick(ctx) { } // The hourly full read, and the first read at startup. comment_count catches comments on tasks the // cursor window missed; a changed count is checked against the comment list before it is reported. +// With tracker.descendants on it reads the project list first, so a new child joins the tree here +// and a child that left it has its tasks read one by one through missing(). export async function reconcile(ctx) { const startMs = ctx.clock(); const readAt = iso(startMs); const sinceMs = ctx.lastReconcile ?? startMs; - const open = await board(ctx); - const list = await walk(ctx, '', { expand: 'comment_count' }); + if (ctx.descendants) await discover(ctx); + const reads = []; + for (const [project, view] of ctx.tree) + reads.push({ project, view, open: await openTasks(ctx, project, view), list: await walk(ctx, '', { expand: 'comment_count' }, project) }); const current = new Map(ctx.broker.taskView(ctx.business, 'current').map((x) => [x.task_ref, x])); const out = { snapshots: [], events: [] }; const listed = new Set(); - for (const t of list) { - listed.add(ref(ctx.project, t.id)); - const bucket = t.done ? ctx.view.done : open.get(t.id)?.bucket; - if (!bucket) continue; - let comments = 0; - const count = t.comment_count; - if (count !== undefined && !Number.isSafeInteger(count)) shape(); - const before = ctx.commentCount.get(t.id); - if (count !== undefined) ctx.commentCount.set(t.id, count); - if (before !== undefined && count !== before) comments = await foreignComments(ctx, t.id, sinceMs); - consider(ctx, current, { task: t, bucket, via: 'reconcile', readAt, comments }, out); - } + for (const { project, view, open, list } of reads) + for (const t of list) { + listed.add(ref(project, t.id)); + const bucket = t.done ? doneBucket(view) : open.get(t.id)?.bucket; + if (bucket === undefined) continue; + let comments = 0; + const count = t.comment_count; + if (count !== undefined && !Number.isSafeInteger(count)) shape(); + const before = ctx.commentCount.get(t.id); + if (count !== undefined) ctx.commentCount.set(t.id, count); + if (before !== undefined && count !== before) comments = await foreignComments(ctx, t.id, sinceMs); + consider(ctx, current, project, { task: t, bucket, via: 'reconcile', readAt, comments }, out); + } const gone = [...current.values()] .filter((x) => x.fields.gone === undefined && !listed.has(x.task_ref)) .map((x) => x.task_ref); diff --git a/packages/tasks/tests/shapes.test.mjs b/packages/tasks/tests/shapes.test.mjs index 3859dd22..c16f1f44 100644 --- a/packages/tasks/tests/shapes.test.mjs +++ b/packages/tasks/tests/shapes.test.mjs @@ -1,7 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; -import { world, sleep } from './world.mjs'; +import { world, sleep, SCOPES } from './world.mjs'; import { taskFields } from '../src/digest.mjs'; import { FakeVikunja } from '../src/fake.mjs'; const recorded = JSON.parse(readFileSync(new URL('./fixtures/v2-shapes.json', import.meta.url))); @@ -154,6 +154,30 @@ test('the fake answers each route with the statuses and shapes Vikunja v2.7.0 se for (const key of new Set(Object.values(KEY))) if (!UNPRODUCED.has(key)) assert.ok(covered.has(key), `the scenario never produced ${key}`); }); +// No recording holds a GET /projects list. Its envelope is the one every list sent, and each item +// is checked against the recorded GET /projects/{p} object, which has parent_project_id and is_archived. +test('the project list holds recorded project objects in the recorded list envelope', async (t) => { + const w = await world(t, { scopes: { sync: { ...SCOPES.sync, projects: [...SCOPES.sync.projects, 'read_all'] } } }); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + w.fake.pseudoProject(w.bots.sync); + const r = await w.fake.fetch('http://vikunja.test/api/v2/projects?per_page=50&page=1', { + headers: { authorization: 'Bearer ' + w.tokens.sync }, + }); + assert.equal(r.status, 200); + const body = await r.json(); + const problems = []; + compare({ ...body, items: [] }, samples['views 200'], 'projects', problems); + for (const p of body.items) compare(p, samples['project 200'], 'projects[]', problems); + assert.deepEqual(problems, []); + assert.deepEqual( + body.items.map((p) => [p.id, p.parent_project_id, p.is_archived]), + [ + [w.project, 0, false], + [child, w.project, false], + [-1, 0, false], + ], + ); +}); test('the recorded task bodies pass the checks S3 applies to every read', () => { const tasks = []; for (const [name, r] of Object.entries(recorded)) { diff --git a/packages/tasks/tests/tree.test.mjs b/packages/tasks/tests/tree.test.mjs new file mode 100644 index 00000000..e6851468 --- /dev/null +++ b/packages/tasks/tests/tree.test.mjs @@ -0,0 +1,493 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { world, sleep, SCOPES } from './world.mjs'; +import { tasksAdapter } from '../src/index.mjs'; +import { WATCH_ONLY } from '../src/sync.mjs'; +// tracker.descendants: the tracker project is the root of a tree, read with its descendants. +// The sync token also needs the project list, projects read_all. +const TREE = { + tracker: { descendants: true }, + scopes: { sync: { ...SCOPES.sync, projects: [...SCOPES.sync.projects, 'read_all'] } }, +}; +const behind = { now: () => Date.now() - 3600000 }; +const state = (w) => w.adapter.status()[0]; +const ready = async (w) => { + await w.adapter.start('demo'); + assert.equal(state(w).state, 'ready', state(w).refused); + return w; +}; +const ref = (project, task) => `vikunja:${project}/${task}`; +const add = async (w, project, title) => (await w.ui('POST', `/projects/${project}/tasks`, { title })).json.id; +const move = (w, project, bucket, task) => + w.ui('PUT', `/projects/${project}/views/${w.fake.kanban(project).id}/buckets/${bucket}/tasks`, { task_id: task }); +const open = (w) => w.broker.taskView('demo', 'open').map((r) => ({ ...r })); +const missing = (w) => w.events('task.missing').map((e) => [e.subject, e.body]); +// The requests one call makes, as "METHOD path". +const counted = async (w, fn) => { + const from = w.fake.requests.length; + await fn(); + return w.fake.requests.slice(from).map((r) => `${r.method} ${r.path}`); +}; +test('with descendants off the adapter reads no project list and no child, as in v0', async (t) => { + const w = await world(t); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + w.fake.install(child); + const task = await add(w, child, 'in the child'); + await ready(w); + await sleep(20); + await w.adapter.tick('demo'); + await w.adapter.reconcile('demo'); + // GETs only: the POST that made the child's task is the test's own. + const paths = w.fake.requests.filter((r) => r.method === 'GET').map((r) => r.path); + assert.ok(!paths.includes('/projects')); + assert.ok(!paths.some((p) => p.startsWith(`/projects/${child}/`))); + assert.equal(w.snapshots(ref(child, task)).length, 0); +}); +test('a three-level tree is read per project, and a project outside it is not', async (t) => { + const w = await world(t, { ...TREE, fake: behind }); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + const childBoard = w.fake.install(child); + // The grandchild keeps Vikunja's default buckets, so it is watch-only. + const grandchild = w.fake.project('Grandchild', w.owner, { parent: child }); + const outside = w.fake.project('Outside', w.owner); + w.fake.share(outside, w.bots.sync, 0); + const tasks = {}; + for (const [name, p] of Object.entries({ root: w.project, child, grandchild, outside })) tasks[name] = await add(w, p, name); + await ready(w); + assert.deepEqual(w.snapshots(ref(w.project, tasks.root)).at(-1).fields.bucket, w.installed.buckets.todo); + assert.deepEqual(w.snapshots(ref(child, tasks.child)).at(-1).fields.bucket, childBoard.buckets.todo); + assert.deepEqual(w.snapshots(ref(grandchild, tasks.grandchild)).at(-1).fields.bucket, WATCH_ONLY); + assert.equal(w.snapshots(ref(outside, tasks.outside)).length, 0); + assert.deepEqual( + open(w).map((r) => r.task_ref), + [ref(w.project, tasks.root), ref(child, tasks.child), ref(grandchild, tasks.grandchild)].sort(), + ); + // Access to the children comes from the root's share alone: the sync bot has no grant on them. + // A quiet tick costs two reads per project: the board (or the open list) and the cursor walk. + const tick = await counted(w, () => w.adapter.tick('demo')); + assert.equal(tick.length, 6); + assert.deepEqual( + tick.map((r) => r.replace(/\/views\/\d+\/buckets\/tasks$/, ' board').replace(/\/tasks$/, ' list')), + [ + `GET /projects/${w.project} board`, + `GET /projects/${w.project} list`, + `GET /projects/${child} board`, + `GET /projects/${child} list`, + `GET /projects/${grandchild} list`, + `GET /projects/${grandchild} list`, + ], + ); + // A reconcile adds the project list and each child's views and buckets. + const reconcile = await counted(w, () => w.adapter.reconcile('demo')); + assert.equal(reconcile[0], 'GET /projects'); + assert.equal(reconcile.length, 1 + 2 * 2 + 3 * 2); +}); +test('a watch-only child keeps the business ready, and a failed child read refuses as transient', async (t) => { + const w = await world(t, TREE); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + w.fake.install(child); + // A second kanban view fails the install checks. + w.fake.addView(child, 'kanban'); + const task = await add(w, child, 'watched'); + await ready(w); + assert.equal(w.snapshots(ref(child, task)).at(-1).fields.bucket, WATCH_ONLY); + const down = await world(t, TREE); + const other = down.fake.project('Child', down.owner, { parent: down.project }); + down.fake.fault('GET', new RegExp(`^/projects/${other}/views$`), 503); + await down.adapter.start('demo'); + assert.equal(state(down).refused, 'tracker-unavailable'); +}); +test('a root without the installed buckets refuses with descendants on', async (t) => { + const w = await world(t, { ...TREE, install: false }); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + w.fake.install(child); + await w.adapter.start('demo'); + assert.equal(state(w).refused, 'tracker-install'); +}); +test('the project list refuses without read_all, and skips archived projects and saved filters', async (t) => { + const w = await world(t, { tracker: { descendants: true } }); + await w.adapter.start('demo'); + assert.equal(state(w).refused, 'tracker-unauthorized'); + // Vikunja leaves archived projects out of the list; this fetch asks for them too, so the + // adapter's own is_archived check is what skips them. + let fake; + const v = await world(t, { + ...TREE, + fetch: (url, init) => fake.fetch(/\/api\/v2\/projects\?/.test(url) ? url + '&is_archived=true' : url, init), + }); + fake = v.fake; + const archived = v.fake.project('Archived', v.owner, { parent: v.project }); + v.fake.install(archived); + const task = await add(v, archived, 'archived'); + v.fake.archive(archived); + v.fake.pseudoProject(v.bots.sync); + await ready(v); + assert.equal(v.snapshots(ref(archived, task)).length, 0); + assert.ok(!v.fake.requests.some((r) => r.method === 'GET' && r.path.startsWith(`/projects/${archived}/`) || r.path.startsWith('/projects/-'))); +}); +test('descendants must be true or false', async (t) => { + const w = await world(t); + await assert.rejects( + tasksAdapter({ trackers: { demo: { baseUrl: 'http://vikunja.test', project: w.project, descendants: 'yes' } }, autostart: false })({ + broker: w.broker, + credentials: w.credentials, + businesses: w.businesses, + }), + (e) => e.code === 'tracker-config', + ); +}); +test('a move inside the tree is a moved tombstone plus a snapshot under the new ref', async (t) => { + const w = await world(t, TREE); + const a = w.fake.project('A', w.owner, { parent: w.project }); + const boardA = w.fake.install(a); + const b = w.fake.project('B', w.owner, { parent: a }); + const outside = w.fake.project('Outside', w.owner); + w.fake.share(outside, w.bots.sync, 0); + const one = await add(w, w.project, 'root to child'); + const two = await add(w, a, 'child to child'); + const three = await add(w, a, 'child to outside'); + await ready(w); + w.fake.moveToProject(one, a); + w.fake.moveToProject(two, b); + w.fake.moveToProject(three, outside); + await sleep(20); + await w.adapter.tick('demo'); + assert.deepEqual(missing(w), [ + [ref(w.project, one), { reason: 'moved', project: a }], + [ref(a, two), { reason: 'moved', project: b }], + [ref(a, three), { reason: 'moved', project: outside }], + ]); + assert.deepEqual(w.snapshots(ref(w.project, one)).at(-1).fields, { gone: 'moved', project: a }); + assert.equal(w.snapshots(ref(a, one)).at(-1).fields.bucket, boardA.buckets.todo); + assert.equal(w.snapshots(ref(b, two)).at(-1).fields.bucket, WATCH_ONLY); + assert.equal(w.snapshots(ref(outside, three)).length, 0); + assert.deepEqual( + open(w).map((r) => r.task_ref), + [ref(a, one), ref(b, two)].sort(), + ); + await sleep(20); + assert.deepEqual(await w.adapter.tick('demo'), { snapshots: 0, events: 0 }); + assert.deepEqual(await w.adapter.reconcile('demo'), { snapshots: 0, events: 0 }); +}); +test('a child made after startup joins the tree at the next reconcile', async (t) => { + const w = await ready(await world(t, TREE)); + const child = w.fake.project('Late', w.owner, { parent: w.project }); + const task = await add(w, child, 'late'); + await sleep(20); + await w.adapter.tick('demo'); + assert.equal(w.snapshots(ref(child, task)).length, 0); + await w.adapter.reconcile('demo'); + assert.equal(w.snapshots(ref(child, task)).at(-1).via, 'reconcile'); + await w.ui('PATCH', `/tasks/${task}`, { title: 'renamed' }); + await sleep(20); + await w.adapter.tick('demo'); + assert.equal(w.snapshots(ref(child, task)).at(-1).fields.title, 'renamed'); +}); +test('a child that leaves the tree has its tasks recorded missing, and the ledger keeps their history', async (t) => { + // The fake's clock runs ahead on demand, so an edit gets a new `updated` second. + let ahead = 0; + const w = await world(t, { ...TREE, fake: { now: () => Date.now() + ahead } }); + const children = {}; + for (const name of ['archived', 'reparented', 'hidden']) { + children[name] = w.fake.project(name, w.owner, { parent: w.project }); + w.fake.install(children[name]); + } + const launchpad = w.fake.project('Launchpad', w.owner); + const tasks = {}; + for (const name of Object.keys(children)) tasks[name] = await add(w, children[name], name); + await ready(w); + // Archived: Vikunja leaves it out of the list. Reparented: shared with the sync bot directly, + // now a top-level project outside the tree. Hidden: moved under a project the bot can't read. + w.fake.archive(children.archived); + w.fake.share(children.reparented, w.bots.sync, 0); + w.fake.reparent(children.reparented, 0); + w.fake.reparent(children.hidden, launchpad); + await sleep(20); + const refs = Object.keys(children).map((name) => ref(children[name], tasks[name])); + const [archived, reparented, hidden] = refs; + // A tick reads the tree it has, and the project list is read at reconcile. The hidden child's + // board answers 403, so the tick drops it at once; the other two still read until the reconcile. + await w.adapter.tick('demo'); + assert.deepEqual(missing(w), [[hidden, { reason: 'no-access' }]]); + // An edit the reconcile reads after the project left keeps the tombstone's `updated` the last one + // the bus recorded. + ahead = 2000; + await w.ui('PATCH', `/tasks/${tasks.reparented}`, { title: 'edited outside' }); + await w.adapter.reconcile('demo'); + assert.deepEqual( + missing(w), + [hidden, archived, reparented].map((r) => [r, { reason: 'no-access' }]), + ); + for (const r of refs) { + const s = w.snapshots(r); + assert.deepEqual( + s.map((x) => x.fields.gone ?? 'open'), + ['open', 'no-access'], + ); + assert.equal(s[1].updated, s[0].updated); + } + assert.ok(!JSON.stringify(w.store.all('SELECT * FROM events')).includes('Launchpad')); + assert.deepEqual(open(w), []); + await sleep(20); + assert.deepEqual(await w.adapter.tick('demo'), { snapshots: 0, events: 0 }); + assert.deepEqual(await w.adapter.reconcile('demo'), { snapshots: 0, events: 0 }); +}); +test('a child deleted or hidden between reconciles drops out at the next tick, and the root still refuses', async (t) => { + const w = await world(t, TREE); + const deleted = w.fake.project('Deleted', w.owner, { parent: w.project }); + w.fake.install(deleted); + // Watch-only, so its tick read is the task list, not a board. + const hidden = w.fake.project('Hidden', w.owner, { parent: w.project }); + const kept = w.fake.project('Kept', w.owner, { parent: w.project }); + const launchpad = w.fake.project('Launchpad', w.owner); + const tasks = {}; + for (const [name, p] of Object.entries({ root: w.project, deleted, hidden, kept })) tasks[name] = await add(w, p, name); + await ready(w); + w.fake.deleteProject(deleted); + w.fake.reparent(hidden, launchpad); + await w.ui('PATCH', `/tasks/${tasks.root}`, { title: 'root renamed' }); + await w.ui('PATCH', `/tasks/${tasks.kept}`, { title: 'kept renamed' }); + await sleep(20); + await w.adapter.tick('demo'); + assert.equal(state(w).state, 'ready'); + assert.deepEqual(missing(w).sort(), [ + [ref(deleted, tasks.deleted), { reason: 'not-found' }], + [ref(hidden, tasks.hidden), { reason: 'no-access' }], + ]); + // The root and the sibling changed in the same tick, and both are recorded. + assert.equal(w.snapshots(ref(w.project, tasks.root)).at(-1).fields.title, 'root renamed'); + assert.equal(w.snapshots(ref(kept, tasks.kept)).at(-1).fields.title, 'kept renamed'); + // The next tick reads neither, and the reconcile finds neither. + const reads = await counted(w, () => w.adapter.tick('demo')); + assert.ok(!reads.some((r) => r.includes(`/projects/${deleted}/`) || r.includes(`/projects/${hidden}/`)), reads.join('\n')); + assert.deepEqual(await w.adapter.reconcile('demo'), { snapshots: 0, events: 0 }); + // Any other failure on a child still refuses the tick, and the child stays. + w.fake.fault('GET', new RegExp(`^/projects/${kept}/tasks$`), 503); + await assert.rejects(w.adapter.tick('demo'), { code: 'tracker-unavailable' }); + await w.ui('PATCH', `/tasks/${tasks.kept}`, { title: 'kept again' }); + await sleep(20); + await w.adapter.tick('demo'); + assert.equal(w.snapshots(ref(kept, tasks.kept)).at(-1).fields.title, 'kept again'); + // The root answering 403 or 404 still refuses the tick, and stays in the tree. + w.fake.unshare(w.project, w.bots.sync); + await assert.rejects(w.adapter.tick('demo'), { code: 'tracker-forbidden' }); + w.fake.share(w.project, w.bots.sync, 0); + await w.ui('PATCH', `/tasks/${tasks.root}`, { title: 'root again' }); + await sleep(20); + await w.adapter.tick('demo'); + assert.equal(w.snapshots(ref(w.project, tasks.root)).at(-1).fields.title, 'root again'); + w.fake.deleteProject(w.project); + await assert.rejects(w.adapter.tick('demo'), { code: 'tracker-not-found' }); +}); +test('a reconcile that fails keeps the previous tree', async (t) => { + const w = await world(t, TREE); + const first = w.fake.project('First', w.owner, { parent: w.project }); + const second = w.fake.project('Second', w.owner, { parent: w.project }); + w.fake.install(second); + const task = await add(w, second, 'second'); + await ready(w); + // The first child's install check fails, so the walk stops before it reaches the second. + w.fake.fault('GET', new RegExp(`^/projects/${first}/views$`), 503); + await assert.rejects(w.adapter.reconcile('demo'), { code: 'tracker-unavailable' }); + await w.ui('PATCH', `/tasks/${task}`, { title: 'renamed' }); + await sleep(20); + await w.adapter.tick('demo'); + assert.equal(w.snapshots(ref(second, task)).at(-1).fields.title, 'renamed'); + assert.deepEqual(missing(w), []); +}); +test("a closed task the cursor missed takes its own project's done bucket", async (t) => { + // An hour behind, so the cursor never returns the close and missing() reads the task. + const w = await world(t, { ...TREE, fake: behind }); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + const childBoard = w.fake.install(child); + const watched = w.fake.project('Watched', w.owner, { parent: w.project }); + const tasks = { child: await add(w, child, 'child'), watched: await add(w, watched, 'watched') }; + await ready(w); + for (const id of Object.values(tasks)) await w.ui('PATCH', `/tasks/${id}`, { done: true }); + await sleep(20); + await w.adapter.tick('demo'); + const child1 = w.snapshots(ref(child, tasks.child)).at(-1); + const watched1 = w.snapshots(ref(watched, tasks.watched)).at(-1); + assert.deepEqual([child1.via, child1.fields.done, child1.fields.bucket], ['task', true, childBoard.buckets.done]); + assert.deepEqual([watched1.via, watched1.fields.done, watched1.fields.bucket], ['task', true, WATCH_ONLY]); + assert.notEqual(childBoard.buckets.done, w.installed.buckets.done); +}); +test("closed tasks read at startup take their own project's done bucket", async (t) => { + const w = await world(t, TREE); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + const childBoard = w.fake.install(child); + const watched = w.fake.project('Watched', w.owner, { parent: w.project }); + const tasks = { child: await add(w, child, 'child'), watched: await add(w, watched, 'watched') }; + for (const id of Object.values(tasks)) await w.ui('PATCH', `/tasks/${id}`, { done: true }); + await ready(w); + const child1 = w.snapshots(ref(child, tasks.child)).at(-1); + const watched1 = w.snapshots(ref(watched, tasks.watched)).at(-1); + assert.deepEqual([child1.via, child1.fields.done, child1.fields.bucket], ['reconcile', true, childBoard.buckets.done]); + assert.deepEqual([watched1.via, watched1.fields.done, watched1.fields.bucket], ['reconcile', true, WATCH_ONLY]); +}); +test("a closed task the cursor returns takes its own project's done bucket", async (t) => { + // Two closes per project: one before the tick, so the open read leaves it out, and one between the + // open read and the cursor walk, so both reads return it. `race` closes the second kind. + const race = new Map(); + const w = await world(t, { + ...TREE, + fetch: async (url, init) => { + const r = await w.fake.fetch(url, init); + for (const [project, id] of race) + if (new URL(url).pathname.startsWith(`/api/v2/projects/${project}/`)) { + race.delete(project); + await w.ui('PATCH', `/tasks/${id}`, { done: true }); + } + return r; + }, + }); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + const childBoard = w.fake.install(child); + const watched = w.fake.project('Watched', w.owner, { parent: w.project }); + const before = { [child]: await add(w, child, 'child'), [watched]: await add(w, watched, 'watched') }; + const during = { [child]: await add(w, child, 'child raced'), [watched]: await add(w, watched, 'watched raced') }; + await ready(w); + for (const id of Object.values(before)) await w.ui('PATCH', `/tasks/${id}`, { done: true }); + for (const [project, id] of Object.entries(during)) race.set(+project, id); + await sleep(20); + await w.adapter.tick('demo'); + assert.equal(race.size, 0); + for (const [project, bucket] of [[child, childBoard.buckets.done], [watched, WATCH_ONLY]]) + for (const id of [before[project], during[project]]) { + const s = w.snapshots(ref(project, id)).at(-1); + assert.deepEqual([s.via, s.fields.done, s.fields.bucket], ['cursor', true, bucket]); + } +}); +test('a task moved out of a child that leaves the tree at the same time is moved, not no-access', async (t) => { + const w = await world(t, TREE); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + w.fake.install(child); + const task = await add(w, child, 'leaving'); + await ready(w); + w.fake.archive(child); + w.fake.moveToProject(task, w.project); + await sleep(20); + await w.adapter.reconcile('demo'); + assert.deepEqual(missing(w), [[ref(child, task), { reason: 'moved', project: w.project }]]); + assert.equal(w.snapshots(ref(w.project, task)).at(-1).fields.title, 'leaving'); +}); +test('a task in a bucket outside the five keeps its raw bucket id', async (t) => { + const w = await ready(await world(t, TREE)); + const backlog = w.fake.bucket(w.installed.view, 'Backlog'); + const task = await add(w, w.project, 'parked'); + await move(w, w.project, backlog, task); + await sleep(20); + await w.adapter.tick('demo'); + assert.equal(w.snapshots(ref(w.project, task)).at(-1).fields.bucket, backlog); + assert.deepEqual(open(w), [{ task_ref: ref(w.project, task), bucket: backlog }]); +}); +// Decision 87: bucket 0 on a watch-only snapshot. Nothing in sync.mjs maps 0 to a bucket or a state. +test('bucket 0 on a watch-only task never becomes a bucket or a state', async (t) => { + const w = await world(t, TREE); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + const buckets = w.fake.bucketIds(w.fake.kanban(child).id); + const task = await add(w, child, 'watched'); + await ready(w); + const r = ref(child, task); + // A move between the child's open buckets changes nothing the adapter records. + await move(w, child, buckets.Doing, task); + await sleep(20); + assert.deepEqual(await w.adapter.tick('demo'), { snapshots: 0, events: 0 }); + // Closed through its done bucket, then reopened: done changes, the bucket stays 0. + await move(w, child, buckets.Done, task); + await sleep(20); + await w.adapter.tick('demo'); + assert.deepEqual(open(w), []); + await move(w, child, buckets['To-Do'], task); + await sleep(20); + await w.adapter.tick('demo'); + assert.deepEqual(open(w), [{ task_ref: r, bucket: WATCH_ONLY }]); + const changes = w.events('task.changed.external').filter((e) => e.subject === r); + assert.equal(changes[0].body.via, 'reconcile'); + assert.deepEqual( + changes.slice(1).map((e) => e.body.changed), + [['done'], ['done']], + ); + assert.ok(w.snapshots(r).every((s) => s.fields.bucket === WATCH_ONLY)); + assert.deepEqual(w.events('task.state'), []); + assert.ok(changes.every((e) => !('state' in e.body))); + // A verb on a child task refuses as it does in v0. + await assert.rejects( + w.call(w.agent('pm'), 'task.assign', { task_ref: r, role: 'coder' }), + (e) => e.code === 'task-project', + ); + // The source has no bucket-to-state table and no write path. + const source = readFileSync(new URL('../src/sync.mjs', import.meta.url), 'utf8'); + assert.ok(!/\.titles\b|\.ids\[/.test(source)); + assert.ok(!/'(PUT|POST|PATCH|DELETE)'/.test(source)); +}); +test('a child whose kanban view is deleted still refuses the tick, and the reconcile reads it watch-only', async (t) => { + const w = await world(t, TREE); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + w.fake.install(child); + const task = await add(w, child, 'child'); + const rootTask = await add(w, w.project, 'root'); + await ready(w); + const rows = () => w.store.get('SELECT (SELECT count(*) FROM events) + (SELECT count(*) FROM task_snapshots) n').n; + // The board answers 404 with code 13001, and the project still reads, so the child hasn't left + // the tree. The tick reads the project, fails, and writes nothing, not even the root's edit. + w.fake.deleteView(w.fake.kanban(child).id); + await w.ui('PATCH', `/tasks/${rootTask}`, { title: 'root renamed' }); + await sleep(20); + const n = rows(); + const reads = await counted(w, () => assert.rejects(w.adapter.tick('demo'), { code: 'tracker-not-found' })); + assert.ok(reads.includes(`GET /projects/${child}`), reads.join('\n')); + assert.equal(rows(), n); + assert.equal(state(w).state, 'ready'); + // A 403 on a child the bot still reads fails the tick too, and the child stays. + w.fake.fault('GET', new RegExp(`^/projects/${child}/views/\\d+/buckets/tasks$`), 403); + await assert.rejects(w.adapter.tick('demo'), { code: 'tracker-forbidden' }); + // The reconcile finds no kanban view and reads the child watch-only. Nothing is missing. + await w.adapter.reconcile('demo'); + assert.deepEqual(missing(w), []); + assert.equal(w.snapshots(ref(child, task)).at(-1).fields.bucket, WATCH_ONLY); + assert.equal(w.snapshots(ref(w.project, rootTask)).at(-1).fields.title, 'root renamed'); + // A 401 on a child read after the root read succeeds fails the tick, whatever the project read says. + w.fake.fault('GET', new RegExp(`^/projects/${child}/tasks$`), 401); + w.fake.fault('GET', new RegExp(`^/projects/${child}$`), 404); + const unauthorized = await counted(w, () => assert.rejects(w.adapter.tick('demo'), { code: 'tracker-unauthorized' })); + assert.ok(!unauthorized.includes(`GET /projects/${child}`), unauthorized.join('\n')); + // The owner's read takes the 404 the tick didn't use. + assert.equal((await w.ui('GET', `/projects/${child}`)).status, 404); + // A 404 on its task list whose project read fails as well keeps the child: the tick refuses. + w.fake.fault('GET', new RegExp(`^/projects/${child}/tasks$`), 404); + w.fake.fault('GET', new RegExp(`^/projects/${child}$`), 503); + await assert.rejects(w.adapter.tick('demo'), { code: 'tracker-not-found' }); + await w.ui('PATCH', `/tasks/${task}`, { title: 'child renamed' }); + await sleep(20); + await w.adapter.tick('demo'); + assert.equal(w.snapshots(ref(child, task)).at(-1).fields.title, 'child renamed'); + assert.deepEqual(missing(w), []); +}); +test('a child dropped at a tick comes back at the next reconcile once the bot reads it again', async (t) => { + const w = await world(t, TREE); + const child = w.fake.project('Child', w.owner, { parent: w.project }); + w.fake.install(child); + const launchpad = w.fake.project('Launchpad', w.owner); + const task = await add(w, child, 'child'); + await ready(w); + const bucket = w.snapshots(ref(child, task)).at(-1).fields.bucket; + w.fake.reparent(child, launchpad); + await w.adapter.tick('demo'); + assert.deepEqual(missing(w), [[ref(child, task), { reason: 'no-access' }]]); + // Back under the root: the tick still leaves it out, and the reconcile finds it again. + w.fake.reparent(child, w.project); + const reads = await counted(w, () => w.adapter.tick('demo')); + assert.ok(!reads.some((r) => r.includes(`/projects/${child}`)), reads.join('\n')); + await w.adapter.reconcile('demo'); + assert.deepEqual( + w.snapshots(ref(child, task)).map((x) => x.fields.gone ?? x.fields.bucket), + [bucket, 'no-access', bucket], + ); + assert.ok(open(w).some((r) => r.task_ref === ref(child, task))); + await w.ui('PATCH', `/tasks/${task}`, { title: 'renamed' }); + await sleep(20); + await w.adapter.tick('demo'); + assert.equal(w.snapshots(ref(child, task)).at(-1).fields.title, 'renamed'); +});