// CHAT-01 history limits (#1507, CHAT-02): at most 100 parts per page, 8 MiB // of serialized UTF-8 per page, 64 blocks per part and 262144 characters per // string. Oversize content splits into fragments and continuation parts; it is // never clipped. Byte limits are measured on the serialized JSON, not on // characters. import { createHash } from "node:crypto"; export const LIMITS = Object.freeze({ parts: 100, pageBytes: 8 * 1024 * 1024, blocks: 64, chars: 262144 }); // Internal budgets that keep any single part well inside one page, so a page // always holds at least one part. export const FRAGMENT_BYTES = 1024 * 1024; export const PART_BYTES = 4 * 1024 * 1024; export const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; // A native value used as a CHAT-01 id. Values that do not fit the id pattern // (OpenAI tool calls such as "call_x|fc_y") map to a stable hash. export function safeId(value) { if (typeof value === "string" && ID.test(value)) return value; return "h-" + createHash("sha256").update(String(value)).digest("hex").slice(0, 40); } // Bytes one code point adds to a JSON string literal. function jsonCost(cp) { if (cp === 0x22 || cp === 0x5c) return 2; if (cp < 0x20) return cp === 8 || cp === 9 || cp === 10 || cp === 12 || cp === 13 ? 2 : 6; if (cp < 0x80) return 1; if (cp < 0x800) return 2; if (cp >= 0xd800 && cp <= 0xdfff) return 6; // lone surrogate, escaped by JSON.stringify if (cp <= 0xffff) return 3; return 4; } // Splits a string into fragments of at most LIMITS.chars code points and // FRAGMENT_BYTES of JSON. Never cuts a surrogate pair. "" is one fragment. export function fragments(str) { const out = []; let start = 0, chars = 0, bytes = 0; for (let i = 0; i < str.length;) { const cp = str.codePointAt(i); const width = cp > 0xffff ? 2 : 1; const cost = jsonCost(cp); if (chars > 0 && (chars + 1 > LIMITS.chars || bytes + cost > FRAGMENT_BYTES)) { out.push(str.slice(start, i)); start = i; chars = 0; bytes = 0; } chars += 1; bytes += cost; i += width; } out.push(str.slice(start)); return out; } const bytesOf = (value) => Buffer.byteLength(JSON.stringify(value), "utf8"); // One native unit (a message, a compaction, a notice) becomes one or more // CHAT-01 entries. `unit.blocks` holds logical blocks: { fields, key, value }, // where `value` is the string that may split and `key` names its field. A // block with key null (an attachment) has no string and is one fragment. export function unitParts(unit, base) { const items = []; unit.blocks.forEach((b, ordinal) => { if (b.key === null) { items.push({ ...b.fields, block: ordinal, fragment: 0, lastFragment: true }); return; } const pieces = fragments(b.value); pieces.forEach((piece, k) => { items.push({ ...b.fields, [b.key]: piece, block: ordinal, fragment: k, lastFragment: k === pieces.length - 1 }); }); }); const groups = []; let current = [], currentBytes = 0; for (const item of items) { const size = bytesOf(item) + 1; if (current.length && (current.length === LIMITS.blocks || currentBytes + size > PART_BYTES)) { groups.push(current); current = []; currentBytes = 0; } current.push(item); currentBytes += size; } groups.push(current); return groups.map((content, part) => ({ version: 2, kind: "entry", id: safeId(`${unit.id}:${part}`), conversation: base.conversation, branch: base.branch, parent: unit.parent, execution: base.execution, nativeEntry: unit.nativeEntry, role: unit.role, request: null, content, part, lastPart: part === groups.length - 1, createdAt: unit.createdAt, message: unit.message, })); } // Takes entries from `start` while the page stays within LIMITS. `shell` is // the page record with an empty `entries` array. export function takePage(entries, sizes, start, shell) { let bytes = bytesOf(shell); let end = start; while (end < entries.length && end - start < LIMITS.parts) { const add = sizes[end] + (end > start ? 1 : 0); if (bytes + add > LIMITS.pageBytes) break; bytes += add; end += 1; } if (end === start && start < entries.length) throw new Error("internal: a single part exceeds the page byte limit"); return end; } export { bytesOf };