// The web tools' fence, tested without the network. A local http server // plays every remote host; an injected resolver decides what each name // resolves to, and an injected request function sends "https" urls to that // server over plain http so the redirect, cap, timeout and html logic run // on real sockets. The address rules themselves are tested directly. import { test, after } from "node:test"; import assert from "node:assert/strict"; import { createServer, request as httpRequest } from "node:http"; import { once } from "node:events"; import { mkdirSync } from "node:fs"; import { join } from "node:path"; import { loadWebConfig, webFetch, webSearch, isPublicAddress, htmlToText, WEB_REFUSAL, WEB_TOOL_NAMES, FETCH_MAX_TEXT_CHARS, SEARCH_MAX_RESULTS } from "../src/web.mjs"; import { loadToolsConfig, createToolSet, enabledToolNames, REFUSAL } from "../src/tools.mjs"; import { makeRoot } from "./helpers.mjs"; const hits = []; const server = createServer((req, res) => { hits.push({ host: req.headers.host, path: req.url, ua: req.headers["user-agent"], cookie: req.headers.cookie, auth: req.headers.authorization, method: req.method }); const u = new URL(req.url, "http://x"); switch (u.pathname) { case "/page": res.writeHead(200, { "content-type": "text/html; charset=utf-8" }); return res.end("Names & things

Hello

one

two <3

"); case "/plain": res.writeHead(200, { "content-type": "text/plain" }); return res.end("just text\r\nline 2\n"); case "/json": res.writeHead(200, { "content-type": "application/json" }); return res.end('{"a":1}'); case "/big": { res.writeHead(200, { "content-type": "text/plain" }); const chunk = Buffer.alloc(1024, 0x61); let n = 0; const push = () => { while (n < 64) { n += 1; if (!res.write(chunk)) return res.once("drain", push); } res.end(); }; return push(); } case "/slow": return setTimeout(() => { res.writeHead(200, { "content-type": "text/plain" }); res.end("late"); }, 2000).unref(); case "/drip": res.writeHead(200, { "content-type": "text/plain" }); res.write("start"); return setTimeout(() => res.end("end"), 2000).unref(); case "/binary": res.writeHead(200, { "content-type": "application/octet-stream" }); return res.end(Buffer.from([0, 1, 2])); case "/pdf": res.writeHead(200, { "content-type": "application/pdf" }); return res.end("%PDF"); case "/missing": res.writeHead(404, { "content-type": "text/html" }); return res.end("

gone

"); case "/hop": res.writeHead(302, { location: `/hop${Number(u.searchParams.get("n") || 0) + 1 > 5 ? "" : `?n=${Number(u.searchParams.get("n") || 0) + 1}`}` }); return res.end(); case "/once": res.writeHead(301, { location: "https://public.example/plain" }); return res.end(); case "/to-private": res.writeHead(302, { location: "https://internal.example/plain" }); return res.end(); case "/to-http": res.writeHead(302, { location: "http://public.example/plain" }); return res.end(); case "/to-ip": res.writeHead(302, { location: "https://127.0.0.1/plain" }); return res.end(); case "/search": { const q = u.searchParams.get("q"); if (u.searchParams.get("format") !== "json") { res.writeHead(403); return res.end("json off"); } if (q === "boom") { res.writeHead(500); return res.end("x"); } if (q === "junk") { res.writeHead(200, { "content-type": "application/json" }); return res.end("not json"); } res.writeHead(200, { "content-type": "application/json" }); const results = []; for (let i = 0; i < 14; i += 1) results.push({ title: ` Result ${i} `, url: `https://r.example/${i}`, content: `snippet\n${i}`, engine: "ddg" }); results.unshift({ title: "bad", url: "javascript:alert(1)" }, { title: "nourl" }, "junk"); return res.end(JSON.stringify({ query: q, results })); } default: res.writeHead(404); return res.end(); } }); server.listen(0, "127.0.0.1"); await once(server, "listening"); const port = server.address().port; after(() => server.close()); // Names: public.example and r.example are "public"; internal.example is // private; rebind.example answers with one public and one private address. const table = { "public.example": [{ address: "203.0.113.10", family: 4 }], "r.example": [{ address: "203.0.113.11", family: 4 }], "internal.example": [{ address: "10.0.0.5", family: 4 }], "rebind.example": [{ address: "203.0.113.12", family: 4 }, { address: "192.168.1.1", family: 4 }], "v6.example": [{ address: "::ffff:10.1.1.1", family: 6 }], }; const deps = { lookup: async (host) => { if (!table[host]) { const e = new Error("ENOTFOUND"); e.code = "ENOTFOUND"; throw e; } return table[host]; }, // "https://host/path" goes to the local server as plain http, with the // Host header kept, so the server sees which host was asked for. The // pinned lookup the tool passes is checked: it must be the vetted address. httpsRequest: (opts) => { assert.ok(opts.lookup, "the tool pins the vetted address"); opts.lookup(opts.hostname, {}, (err, address) => { assert.equal(err, null); assert.equal(address, table[opts.hostname][0].address); }); return httpRequest({ ...opts, hostname: "127.0.0.1", port, servername: undefined, lookup: undefined }); }, httpRequest, }; const config = () => loadWebConfig({ searxng: `http://127.0.0.1:${port}`, maxFetchBytes: 16384 }); const fast = () => ({ ...config(), timeoutMs: 500 }); const refuses = (p, reason) => assert.rejects(p, (err) => err.reason === reason, reason); test("web: config takes an https or loopback-http SearXNG base url and a bounded fetch cap", () => { assert.deepEqual(config(), { searxng: `http://127.0.0.1:${port}`, maxFetchBytes: 16384, timeoutMs: 15000 }); assert.equal(loadWebConfig({ searxng: "https://search.example/" }).searxng, "https://search.example"); assert.equal(loadWebConfig({ searxng: "http://localhost:8888" }).maxFetchBytes, 1048576); assert.throws(() => loadWebConfig({ searxng: "http://search.example" }), /https, or http on loopback/); assert.throws(() => loadWebConfig({ searxng: "http://127.0.0.1:8888/search?q=x" }), /bare base url/); assert.throws(() => loadWebConfig({ searxng: "https://u:p@search.example" }), /bare base url/); assert.throws(() => loadWebConfig({ searxng: "nope" }), /not a valid url/); assert.throws(() => loadWebConfig({ searxng: "https://s.example", maxFetchBytes: 100 }), /maxFetchBytes/); assert.throws(() => loadWebConfig({ searxng: "https://s.example", key: "x" }), /unknown key/); assert.throws(() => loadWebConfig({}), /searxng/); assert.deepEqual(WEB_TOOL_NAMES, ["web_fetch", "web_search"]); }); test("web: address rules refuse every private, loopback, link-local, mapped and multicast form", () => { for (const a of ["203.0.113.1", "8.8.8.8", "172.32.0.1", "100.128.0.1", "2606:4700::1111", "::ffff:8.8.8.8"]) assert.equal(isPublicAddress(a), true, a); for (const a of ["10.1.1.1", "127.0.0.1", "127.9.9.9", "0.0.0.0", "169.254.1.1", "172.16.0.1", "172.31.255.255", "192.168.0.1", "192.0.0.1", "100.64.0.1", "198.18.0.1", "224.0.0.1", "255.255.255.255", "::1", "::", "::ffff:10.0.0.1", "::ffff:127.0.0.1", "fd00::1", "fc00::1", "fe80::1", "fec0::1", "ff02::1", "64:ff9b::a00:1", "not-an-ip", "999.1.1.1"]) assert.equal(isPublicAddress(a), false, a); }); test("web: web_fetch refuses bad urls, private hosts, rebinding names, non-https redirects, too many hops, error status, non-text bodies, and times out", async () => { const c = fast(); for (const url of ["http://public.example/page", "ftp://public.example/x", "public.example/page", "https://u:p@public.example/page", "", 5, "https://", "javascript:alert(1)"]) await refuses(webFetch(c, { url }, deps), WEB_REFUSAL.BAD_URL); await refuses(webFetch(c, { url: "https://internal.example/page" }, deps), WEB_REFUSAL.PRIVATE); await refuses(webFetch(c, { url: "https://rebind.example/page" }, deps), WEB_REFUSAL.PRIVATE); await refuses(webFetch(c, { url: "https://v6.example/page" }, deps), WEB_REFUSAL.PRIVATE); await refuses(webFetch(c, { url: "https://127.0.0.1/page" }, deps), WEB_REFUSAL.PRIVATE); await refuses(webFetch(c, { url: "https://[::1]/page" }, deps), WEB_REFUSAL.PRIVATE); await refuses(webFetch(c, { url: "https://10.0.0.1/page" }, deps), WEB_REFUSAL.PRIVATE); await refuses(webFetch(c, { url: "https://nowhere.example/page" }, deps), WEB_REFUSAL.UNRESOLVED); const before = hits.length; await refuses(webFetch(c, { url: "https://public.example/to-private" }, deps), WEB_REFUSAL.PRIVATE); await refuses(webFetch(c, { url: "https://public.example/to-ip" }, deps), WEB_REFUSAL.PRIVATE); await refuses(webFetch(c, { url: "https://public.example/to-http" }, deps), WEB_REFUSAL.BAD_REDIRECT); assert.equal(hits.slice(before).filter((h) => h.host !== "public.example").length, 0, "a refused redirect target is never requested"); await refuses(webFetch(c, { url: "https://public.example/hop" }, deps), WEB_REFUSAL.REDIRECTS); await refuses(webFetch(c, { url: "https://public.example/missing" }, deps), WEB_REFUSAL.STATUS); await refuses(webFetch(c, { url: "https://public.example/binary" }, deps), WEB_REFUSAL.NOT_TEXT); await refuses(webFetch(c, { url: "https://public.example/pdf" }, deps), WEB_REFUSAL.NOT_TEXT); await refuses(webFetch(c, { url: "https://public.example/slow" }, deps), WEB_REFUSAL.TIMEOUT); await refuses(webFetch(c, { url: "https://public.example/drip" }, deps), WEB_REFUSAL.TIMEOUT); for (const h of hits) { assert.equal(h.method, "GET"); assert.match(h.ua, /^mosaic-discord-sage\//); assert.equal(h.cookie, undefined); assert.equal(h.auth, undefined); } }); test("web: web_fetch returns html as text with the title, follows an https redirect, keeps plain text and json, and cuts at the cap", async () => { const c = fast(); const page = await webFetch(c, { url: "https://public.example/page?x=1" }, deps); assert.equal(page.status, 200); assert.equal(page.contentType, "text/html"); assert.equal(page.title, "Names & things"); assert.equal(page.text, "Hello\none\ntwo <3"); assert.equal(page.redirects, 0); assert.equal(page.truncated, false); const hopped = await webFetch(c, { url: "https://public.example/once" }, deps); assert.equal(hopped.finalUrl, "https://public.example/plain"); assert.equal(hopped.redirects, 1); assert.equal(hopped.text, "just text\nline 2"); const j = await webFetch(c, { url: "https://public.example/json" }, deps); assert.equal(j.contentType, "application/json"); assert.equal(j.text, '{"a":1}'); const big = await webFetch(c, { url: "https://public.example/big" }, deps); assert.equal(big.truncated, true); assert.equal(big.bytes, 16384); assert.equal(big.textTruncated, true); assert.equal(big.text.length, FETCH_MAX_TEXT_CHARS); }); test("web: html to text drops scripts, styles and comments, decodes entities and keeps block breaks", () => { const r = htmlToText(" A – B

x
y

1

2

A "q"

"); assert.equal(r.title, "A – B"); assert.equal(r.text, "x\ny\n1 2\n\nA \"q\""); }); test("web: web_search asks the instance for json, returns at most ten clean results, and refuses a bad query, a down instance or an unusable answer", async () => { const c = fast(); const r = await webSearch(c, { query: " content engine name " }, deps); assert.equal(r.query, "content engine name"); assert.equal(r.results.length, SEARCH_MAX_RESULTS); assert.equal(r.total, 17); assert.deepEqual(r.results[0], { title: "Result 0", url: "https://r.example/0", snippet: "snippet 0" }); const last = hits[hits.length - 1]; assert.equal(last.path, "/search?q=content+engine+name&format=json"); await refuses(webSearch(c, { query: "" }, deps), WEB_REFUSAL.BAD_QUERY); await refuses(webSearch(c, { query: "x".repeat(401) }, deps), WEB_REFUSAL.BAD_QUERY); await refuses(webSearch(c, { query: 7 }, deps), WEB_REFUSAL.BAD_QUERY); await refuses(webSearch(c, { query: "boom" }, deps), WEB_REFUSAL.SEARCH_DOWN); await refuses(webSearch(c, { query: "junk" }, deps), WEB_REFUSAL.SEARCH_BAD); await refuses(webSearch({ ...c, searxng: "http://127.0.0.1:1" }, { query: "x" }, deps), WEB_REFUSAL.SEARCH_DOWN); }); test("web: the tool set enables the web tools only with a web key, counts them in the budget, and records url, status and hits", async () => { const base = makeRoot(); const root = join(base, "docs"); mkdirSync(root); const plain = loadToolsConfig({ roots: [{ name: "docs", path: root }] }); assert.deepEqual(enabledToolNames(plain), ["list_dir", "read_file", "search"]); assert.throws(() => createToolSet(plain).call("web_fetch", { url: "https://public.example/page" }), /unknown tool/); assert.throws(() => loadToolsConfig({ roots: [{ name: "docs", path: root }], web: { searxng: "http://evil.example" } }), /loopback/); const cfg = loadToolsConfig({ roots: [{ name: "docs", path: root, write: true }], maxCallsPerTurn: 2, web: { searxng: `http://127.0.0.1:${port}` } }); assert.deepEqual(enabledToolNames(cfg), ["list_dir", "read_file", "search", "write_file", "edit_file", "web_fetch", "web_search"]); assert.deepEqual(cfg.web, { searxng: `http://127.0.0.1:${port}`, maxFetchBytes: 1048576, timeoutMs: 15000 }); const set = createToolSet(cfg); // The real https path would need a real host; the set's call goes through // the default transport, so only the refusals that happen before any // socket are exercised here. The transport itself is covered above. const bad = await set.call("web_fetch", { url: "http://public.example/page" }); assert.equal(bad.ok, false); assert.equal(bad.text, `refused: ${WEB_REFUSAL.BAD_URL}`); assert.deepEqual({ ...bad.details, ms: 0 }, { tool: "web_fetch", root: null, path: null, url: "http://public.example/page", ok: false, reason: WEB_REFUSAL.BAD_URL, ms: 0 }); const s = await set.call("web_search", { query: "content engine" }); assert.equal(s.ok, true); assert.match(s.text, /^10 result\(s\) for "content engine" \(of 17\)\n1\. Result 0\n https:\/\/r\.example\/0\n snippet 0\n/); assert.deepEqual({ ...s.details, ms: 0 }, { tool: "web_search", root: null, path: null, query: "content engine", ok: true, hits: 10, ms: 0 }); assert.equal(set.calls, 2); const over = await set.call("web_search", { query: "again" }); assert.equal(over.details.reason, REFUSAL.BUDGET); });