Files

485 lines
24 KiB
JavaScript

// Starter packs SP8 (#1873): saving an organization, or one of its projects, as a template. Design:
// starter-packs-design section 5 and SP8 in section 8.
//
// A saved template is a pack document (packVersion 1) written to
// <configDir>/templates/<account>/<name>.json and read later with origin `saved`
// (plugins/src/starter-pack.mjs readSavedTemplate). Save copies what section 5 captures and nothing
// else: definitions (role, instructions, personality, from), one hire proposal per agent, the board,
// the workflows, the union of the captured roles' skills and {id, version} for each pack applied
// before. Everything else stays in the organization: members, identities, credential material
// (instance credentials, copyBindings, copyId, orgIncarnation, generation), launch, plugins, vars,
// project roots and git state, definition revs and hire decisions, and everything on the bus.
//
// Free text is kept, with the organization id, its project ids, its agent names, the owner's
// account and the project roots replaced by {org}, {project}, {agent}, {owner} and {root}. The
// preview lists each replacement and every kept free-text field by path, and the exact JSON save
// writes; save takes that JSON's digest. Every string and key is passed to the credential store's
// assertClean; a hit refuses template-secret-suspected naming the path, never the text.
//
// A refusal is a TemplateError: a bus code and a reason that names a path or a checked name only,
// never a value from the document and never a host path.
import { closeSync, constants, fchmodSync, fsyncSync, lstatSync, mkdirSync, openSync, readdirSync, unlinkSync, writeSync } from "node:fs";
import { join } from "node:path";
import { ORG_PREFIX, checkDefinition, readTemplate } from "./definitions.mjs";
import { sha256 } from "./util.mjs";
import { ID_PATTERN } from "./vocabulary.mjs";
import { NAME } from "../../plugins/src/kinds.mjs";
import {
PACK_BOUNDS, PACK_VERSION, SAVED_MAX_BYTES, checkPack, narrowsTemplate, readSavedTemplate,
} from "../../plugins/src/starter-pack.mjs";
// A template name is a pack id (NAME), at most 64 characters, the length of every other pack name.
export const TEMPLATE_NAME_MAX = 64;
// list reads at most this many files; a longer directory refuses rather than truncating silently.
export const TEMPLATE_LIST_MAX = 256;
const CHIEF = "chief-of-staff";
const ASSISTANT = "assistant";
const HIRE_DEFINITION = /^hire-/;
// The board a template carries when the organization (or project) has none: the five task states in
// store order, with plain labels, as a kanban. A pack must carry a board (SP1 checkPack).
export const DEFAULT_BOARD = Object.freeze({
columns: Object.freeze([
Object.freeze({ state: "todo", label: "To do" }),
Object.freeze({ state: "in-progress", label: "In progress" }),
Object.freeze({ state: "in-review", label: "In review" }),
Object.freeze({ state: "blocked", label: "Blocked" }),
Object.freeze({ state: "done", label: "Done" }),
]),
layout: "kanban",
});
const HIRE_REASON = "The organization this template was saved from ran this agent.";
// A word boundary for placeholder replacement: an identifier character on either side means the
// literal is part of a longer name, which stays as it is.
const WORD = "A-Za-z0-9_-";
export class TemplateError extends Error {
constructor(code, reason) {
super(reason ? `${code}: ${reason}` : code);
this.name = "TemplateError";
this.code = code;
this.reason = reason;
}
}
const refuse = (code, reason) => {
throw new TemplateError(code, reason);
};
const plain = (value) => value !== null && typeof value === "object" && !Array.isArray(value)
&& [Object.prototype, null].includes(Object.getPrototypeOf(value));
const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
// The account (the caller's s.human) and the name are checked before any path is built from them.
export function checkAccount(account) {
if (typeof account !== "string" || !ID_PATTERN.test(account)) refuse("human-required", "the saving account is not a valid account name");
return account;
}
export function checkTemplateName(name) {
if (typeof name !== "string" || name.length > TEMPLATE_NAME_MAX || !NAME.test(name)) {
refuse("invalid-request", `a template name must match ${NAME}, at most ${TEMPLATE_NAME_MAX} characters`);
}
return name;
}
// One template, read now, or null when the name is no usable version 2 template.
function templateOrNull(rolesDir, name, cache) {
if (!cache.has(name)) {
let value = null;
if (typeof name === "string" && ID_PATTERN.test(name)) {
try {
value = readTemplate(rolesDir, name);
} catch (error) {
if (error?.name !== "BusinessError") throw error;
}
}
cache.set(name, value);
}
return cache.get(name);
}
// ---- Placeholders ----
// The literals section 5 replaces, longest first so a root path is replaced before the ids inside
// it. A name that is also a template or a definition name is a role's name, not a person's or a
// client's, and stays: replacing "coder" would rewrite every role text that says coder.
function literals(document, { rolesDir, cache }) {
const out = [];
const add = (text, placeholder) => {
if (typeof text === "string" && text.length > 0 && !out.some((l) => l.text === text)) out.push({ text, placeholder });
};
for (const project of Object.values(document.projects ?? {})) {
add(project?.root, "{root}");
add(project?.repo, "{root}");
}
add(document.id, "{org}");
for (const id of Object.keys(document.projects ?? {})) add(id, "{project}");
add(document.human, "{owner}");
const role = (name) => Object.hasOwn(document.definitions ?? {}, name) || templateOrNull(rolesDir, name, cache) !== null;
const agents = new Set([...Object.keys(document.roles ?? {}), ...(Array.isArray(document.retired) ? document.retired : [])]);
for (const project of Object.values(document.projects ?? {})) for (const name of Object.keys(project?.agents ?? {})) agents.add(name);
for (const name of agents) if (!role(name)) add(name, "{agent}");
out.sort((a, b) => b.text.length - a.text.length);
return out;
}
// Returns the text with every literal replaced, and counts each placeholder used. One pass over the
// text with one alternation, longest literal first: a placeholder written in never matches a later
// literal, so an instance named "root" or "agent" cannot rewrite {root} or {agent}.
const combined = new WeakMap();
function replace(text, list, counts) {
if (list.length === 0) return text;
let entry = combined.get(list);
if (!entry) {
const by = new Map(list.map((l) => [l.text, l.placeholder]));
entry = { by, pattern: new RegExp(`(?<![${WORD}])(?:${list.map((l) => escape(l.text)).join("|")})(?![${WORD}])`, "g") };
combined.set(list, entry);
}
return text.replace(entry.pattern, (match) => {
const placeholder = entry.by.get(match);
counts[placeholder] = (counts[placeholder] ?? 0) + 1;
return placeholder;
});
}
// ---- Capture ----
// Builds the template document. With list null, free text is kept as it is (the scan's second
// copy); otherwise each replacement and each kept free-text field is recorded by path.
function capture(document, { name, project, rolesDir, list, record }) {
const cache = new Map();
const text = (value, path) => {
if (list === null) return value;
const counts = {};
const out = replace(value, list, counts);
for (const [placeholder, count] of Object.entries(counts)) record.replacements.push({ path, placeholder, count });
record.freeText.push({ path, text: out });
return out;
};
const orgDefinitions = plain(document.definitions) ? document.definitions : {};
const scope = project === undefined ? "org" : "project";
let source;
if (scope === "project") {
if (!plain(document.projects) || !Object.hasOwn(document.projects, project)) refuse("invalid-request", "the project is not declared in this organization");
source = document.projects[project];
if (source?.kind === "library") refuse("invalid-request", "a library project has no agents, board or workflows to save");
} else {
source = document;
}
// The agents: one hire per org instance, or per project agent that defines one. Never the chief
// or the assistant, which every organization already has.
const agentEntries = scope === "org"
? Object.entries(plain(document.roles) ? document.roles : {})
: Object.entries(plain(source.agents) ? source.agents : {}).filter(([, e]) => typeof e?.definition === "string");
agentEntries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
const agents = [];
for (const [, entry] of agentEntries) {
const definition = entry?.definition;
if (typeof definition !== "string") continue;
const org = definition.startsWith(ORG_PREFIX) ? definition.slice(ORG_PREFIX.length) : null;
const base = org === null ? definition : orgDefinitions[org]?.from?.template;
if (base === CHIEF || base === ASSISTANT) continue;
if (org !== null && !Object.hasOwn(orgDefinitions, org)) refuse("invalid-request", "an agent names an organization definition the file does not hold");
agents.push({ org, template: org === null ? definition : null });
}
// Which org definitions are copied: every one for an org template; for a project template the
// ones its agents and workflows name.
const sourceWorkflows = plain(source.workflows) ? source.workflows : {};
let copied;
if (scope === "org") {
copied = Object.keys(orgDefinitions);
} else {
const named = new Set(agents.filter((a) => a.org !== null).map((a) => a.org));
for (const workflow of Object.values(sourceWorkflows)) {
for (const step of Array.isArray(workflow?.steps) ? workflow.steps : []) {
for (const role of Array.isArray(step?.roles) ? step.roles : []) if (Object.hasOwn(orgDefinitions, role)) named.add(role);
}
}
copied = Object.keys(orgDefinitions).filter((n) => named.has(n));
}
// Names. A definition named hire-<decision> becomes <from>-<n>, so no decision id leaves the
// organization; every other copied definition keeps its name.
const taken = new Set(copied.filter((n) => !HIRE_DEFINITION.test(n)));
const free = (base) => {
if (!taken.has(base)) { taken.add(base); return base; }
for (let n = 1; ; n++) if (!taken.has(`${base}-${n}`)) { taken.add(`${base}-${n}`); return `${base}-${n}`; }
};
const numbered = (base) => {
for (let n = 1; ; n++) if (!taken.has(`${base}-${n}`)) { taken.add(`${base}-${n}`); return `${base}-${n}`; }
};
const renamed = new Map();
for (const n of copied) {
if (!HIRE_DEFINITION.test(n)) { renamed.set(n, n); continue; }
renamed.set(n, numbered(orgDefinitions[n]?.from?.template ?? "role"));
}
// Definitions: role, instructions, personality and from. No rev.
const definitions = {};
const skills = [];
const addSkills = (list) => { for (const s of Array.isArray(list) ? list : []) if (!skills.includes(s)) skills.push(s); };
for (const n of copied) {
const entry = orgDefinitions[n];
const key = renamed.get(n);
const at = `definitions.${key}`;
if (!plain(entry) || !plain(entry.role)) refuse("invalid-request", `${at} is not a definition entry`);
const out = {};
if (entry.from !== undefined) {
const template = templateOrNull(rolesDir, entry.from?.template, cache);
if (!template) refuse("starter-pack-invalid", `${at}.from names a template that is not a version 2 role`);
// A template that changed since the definition copied it: the saved from names its current
// hash, and the role must still narrow the current template. The preview lists the re-pin.
if (template.sha256 !== entry.from.sha256 && list !== null) record.repinned.push(`${at}.from`);
out.from = { template: template.name, sha256: template.sha256 };
let composition;
try {
({ composition } = checkDefinition(key, { ...entry, rev: 1, from: out.from }, { where: "definitions" }));
} catch (error) {
if (error?.name === "BusinessError") refuse("starter-pack-invalid", `${at} fails the composition check`);
throw error;
}
if (!narrowsTemplate(composition, template.role)) refuse("starter-pack-invalid", `${at}.role is wider than template ${template.name}`);
}
out.role = structuredClone(entry.role);
if (typeof entry.instructions === "string") out.instructions = text(entry.instructions, `${at}.instructions`);
if (entry.personality !== undefined) {
const p = structuredClone(entry.personality);
if (plain(p)) {
if (typeof p.summary === "string") p.summary = text(p.summary, `${at}.personality.summary`);
if (typeof p.voice === "string") p.voice = text(p.voice, `${at}.personality.voice`);
if (Array.isArray(p.traits)) p.traits = p.traits.map((t, i) => (typeof t === "string" ? text(t, `${at}.personality.traits[${i}]`) : t));
}
out.personality = p;
}
definitions[key] = out;
addSkills(entry.role.skills);
}
// Hires: <from>-<n> and the definition. A template agent hires a template reference; an org
// definition agent hires the copied definition.
const refs = new Map();
const hireNames = new Set();
const hires = [];
for (const agent of agents) {
let definition;
let base;
if (agent.template !== null) {
const template = templateOrNull(rolesDir, agent.template, cache);
if (!template) refuse("starter-pack-invalid", `an agent uses template ${agent.template}, which is not a version 2 role`);
if (!refs.has(agent.template)) {
const key = free(agent.template);
definitions[key] = { template: template.name, sha256: template.sha256 };
refs.set(agent.template, key);
addSkills(template.role.skills);
}
definition = refs.get(agent.template);
base = agent.template;
} else {
definition = renamed.get(agent.org);
base = orgDefinitions[agent.org]?.from?.template ?? definition;
}
let hireName;
for (let n = 1; ; n++) if (!hireNames.has(`${base}-${n}`)) { hireName = `${base}-${n}`; break; }
hireNames.add(hireName);
hires.push({ name: hireName, definition, reason: HIRE_REASON });
}
if (hires.length > PACK_BOUNDS.hires) refuse("starter-pack-invalid", `a template holds at most ${PACK_BOUNDS.hires} hires; this one would hold ${hires.length}`);
// The board, or the default when there is none.
const sourceBoard = plain(source.board) ? source.board : DEFAULT_BOARD;
const board = {
columns: sourceBoard.columns.map((c, i) => ({ state: c.state, label: text(c.label, `board.columns[${i}].label`) })),
layout: sourceBoard.layout,
};
// Workflows. A pack step carries an id; the organization's steps have none, so save numbers them.
// A pack workflow has no title: a workflow without a description takes its title as one.
const workflows = {};
for (const [wname, workflow] of Object.entries(sourceWorkflows)) {
const at = `workflows.${wname}`;
const description = typeof workflow?.description === "string" ? text(workflow.description, `${at}.description`)
: typeof workflow?.title === "string" ? text(workflow.title, `${at}.description`)
: `The ${wname} workflow.`;
const steps = (Array.isArray(workflow?.steps) ? workflow.steps : []).map((step, i) => ({
id: `step-${i + 1}`,
title: text(step.title, `${at}.steps[${i}].title`),
roles: (Array.isArray(step.roles) ? step.roles : []).map((r) => (renamed.has(r) ? renamed.get(r) : r)),
state: step.state,
}));
workflows[wname] = { description, steps };
}
// packs: {id, version} for each pack applied before at this scope, once each, the latest 32.
const seen = new Set();
const packs = [];
for (const entry of Array.isArray(document.packs) ? document.packs : []) {
if (scope === "org" ? entry?.scope !== "org" : entry?.project !== project) continue;
const k = `${entry.id}\n${entry.version}`;
if (seen.has(k)) continue;
seen.add(k);
packs.push({ id: entry.id, version: entry.version });
}
if (list !== null) record.packsDropped = Math.max(0, packs.length - PACK_BOUNDS.packs);
const keptPacks = packs.slice(Math.max(0, packs.length - PACK_BOUNDS.packs));
const label = name.slice(0, PACK_BOUNDS.label);
const descriptionText = scope === "org" ? "Saved from {org}." : "Saved from {project} in {org}.";
return {
packVersion: PACK_VERSION, id: name, version: "1.0.0", label, description: descriptionText, scope,
definitions, hires, board, workflows, skills, packs: keptPacks,
};
}
// Every string and every key, by path, through the credential store's check. A hit refuses
// template-secret-suspected and names the path; the text never leaves.
function scan(value, path, assertClean) {
const check = (v, where) => {
try {
assertClean(v);
} catch (error) {
if (error?.code === "credential-leak") refuse("template-secret-suspected", where);
throw error;
}
};
if (typeof value === "string") return check(value, path);
if (Array.isArray(value)) return value.forEach((v, i) => scan(v, `${path}[${i}]`, assertClean));
if (value !== null && typeof value === "object") {
for (const [k, v] of Object.entries(value)) {
const where = path ? `${path}.${k}` : k;
check(k, `${path || "template"}[key]`);
scan(v, where, assertClean);
}
}
}
// The preview: the exact JSON save writes, its digest, each replacement and each kept free-text
// field by path. Refuses before returning when any string or key holds a held credential value.
export function templatePreview({ document, name, project, rolesDir, assertClean }) {
checkTemplateName(name);
if (project !== undefined && (typeof project !== "string" || !ID_PATTERN.test(project))) refuse("invalid-request", "the project id is invalid");
if (typeof assertClean !== "function") throw new TypeError("templatePreview needs the credential store's assertClean");
if (!plain(document)) refuse("invalid-request", "the organization file is not readable");
const scope = { name, project, rolesDir };
// The scan reads both copies: the kept text before replacement (a held value a placeholder would
// break up still refuses) and the document save writes.
const raw = capture(document, { ...scope, list: null, record: null });
scan(raw, "", assertClean);
const record = { replacements: [], freeText: [], repinned: [], packsDropped: 0 };
const template = capture(document, { ...scope, list: literals(document, { rolesDir, cache: new Map() }), record });
scan(template, "", assertClean);
let checked;
try {
checked = checkPack(template, { origin: "saved", rolesDir });
} catch (error) {
if (error?.name === "PluginError") {
refuse(/^starter-pack-[a-z-]+$/.test(error.code ?? "") ? error.code : "starter-pack-invalid", typeof error.message === "string" ? error.message.slice(0, 300) : undefined);
}
throw error;
}
const json = `${JSON.stringify(checked, null, 2)}\n`;
if (Buffer.byteLength(json) > SAVED_MAX_BYTES) refuse("starter-pack-invalid", `a saved template is at most ${SAVED_MAX_BYTES} bytes`);
const byPath = (a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
return {
name, scope: checked.scope, ...(project === undefined ? {} : { project }),
template: checked, json, digest: sha256(json),
replacements: record.replacements.sort(byPath), freeText: record.freeText.sort(byPath),
repinned: record.repinned.sort(), packsDropped: record.packsDropped,
};
}
// ---- The file ----
// A directory save writes under: real, the uid's own, writable by nobody else. The account's own
// directory is private (0700 or narrower), as readSavedTemplate requires.
function directory(path, uid, { private: closed }) {
try {
mkdirSync(path, { mode: 0o700 });
} catch (error) {
if (error?.code !== "EEXIST") refuse("not-configured", `the ${closed ? "account" : "templates"} directory cannot be created`);
}
let st;
try {
st = lstatSync(path);
} catch {
refuse("not-configured", `the ${closed ? "account" : "templates"} directory is unreadable`);
}
if (!st.isDirectory() || st.isSymbolicLink() || st.uid !== uid || (st.mode & 0o022) !== 0 || (closed && (st.mode & 0o077) !== 0)) {
refuse("not-configured", `the ${closed ? "account" : "templates"} directory is not private to this instance's user`);
}
}
// Save: the preview again, its digest compared, then the file created exclusively (wx, 0600). A
// name already saved refuses; nothing is replaced.
export function templateSave({ document, name, project, digest, rolesDir, assertClean, configDir, account, uid = process.getuid() }) {
checkAccount(account);
checkTemplateName(name);
if (typeof digest !== "string" || !/^[0-9a-f]{64}$/.test(digest)) refuse("invalid-request", "save takes the preview's digest");
const preview = templatePreview({ document, name, project, rolesDir, assertClean });
if (preview.digest !== digest) refuse("pack-plan-stale", "the organization or a template changed since the preview");
const templates = join(configDir, "templates");
const home = join(templates, account);
directory(templates, uid, { private: false });
directory(home, uid, { private: true });
const file = join(home, `${name}.json`);
let fd;
try {
fd = openSync(file, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
} catch (error) {
if (error?.code === "EEXIST") refuse("invalid-request", "a template of this name is already saved");
refuse("not-configured", "the template file cannot be created");
}
try {
fchmodSync(fd, 0o600);
const bytes = Buffer.from(preview.json);
let offset = 0;
while (offset < bytes.length) offset += writeSync(fd, bytes, offset);
fsyncSync(fd);
} catch (error) {
// A partial file would hold the name: remove the file this call created, then report the failure.
const open = fd;
fd = undefined;
try { closeSync(open); } catch { /* the write failure is the one reported */ }
try { unlinkSync(file); } catch { /* already gone */ }
throw error;
} finally {
if (fd !== undefined) closeSync(fd);
}
return { name, scope: preview.scope, ...(project === undefined ? {} : { project }), digest };
}
// List: the caller's own saved templates, each read through readSavedTemplate. A file that does not
// read is listed as unreadable, by name only. No account directory yet is an empty list.
export function templateList({ configDir, account, rolesDir, uid = process.getuid() }) {
checkAccount(account);
const home = join(configDir, "templates", account);
let names;
try {
const st = lstatSync(home);
if (!st.isDirectory() || st.isSymbolicLink() || st.uid !== uid || (st.mode & 0o077) !== 0) {
refuse("not-configured", "the account directory is not private to this instance's user");
}
names = readdirSync(home);
} catch (error) {
if (error instanceof TemplateError) throw error;
if (error?.code === "ENOENT") return { templates: [] };
refuse("not-configured", "the account directory is unreadable");
}
const files = names.filter((f) => f.endsWith(".json")).map((f) => f.slice(0, -".json".length))
.filter((n) => n.length <= TEMPLATE_NAME_MAX && NAME.test(n)).sort();
if (files.length > TEMPLATE_LIST_MAX) refuse("invalid-request", `the account holds more than ${TEMPLATE_LIST_MAX} saved templates`);
const templates = files.map((n) => {
try {
const { pack } = readSavedTemplate({ configDir, account, name: n, rolesDir, uid });
return { name: n, version: pack.version, label: pack.label, description: pack.description, scope: pack.scope, hires: pack.hires.length };
} catch (error) {
if (error?.name === "PluginError") return { name: n, readable: false, code: error.code };
throw error;
}
});
return { templates };
}