Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
fix(llm): refuse local hosts in read_web_page before fetching
Elian Doran89613e98506c
diff --git a/packages/trilium-core/src/services/llm/web_search.spec.ts b/packages/trilium-core/src/services/llm/web_search.spec.tsindex 67aa49bbfa..e0997c1cb0 100644--- a/packages/trilium-core/src/services/llm/web_search.spec.ts+++ b/packages/trilium-core/src/services/llm/web_search.spec.ts@@ -166,6 +166,26 @@ describe("readWebPage", () => {await expect(readWebPage("https://a.example/doc.pdf")).rejects.toThrow("application/pdf");});+ it("refuses a local or private-network host before fetching, on every runtime", async () => {+ const local = [+ "http://localhost:8080/", "http://app.localhost/", "http://router/", "http://nas.local/",+ "http://box.lan/", "http://db.internal/", "http://printer.home.arpa/", "http://localhost./",+ "http://127.0.0.1/", "http://2130706433/", "http://0.0.0.0/", "http://10.1.2.3/",+ "http://172.16.0.1/", "http://172.31.255.255/", "http://192.168.1.1/", "http://169.254.169.254/",+ "http://100.64.0.1/", "http://[::1]/", "http://[::]/", "http://[fd00::1]/", "http://[fe80::1]/",+ "http://[::ffff:127.0.0.1]/", "http://[::ffff:192.168.0.1]/"+ ];+ for (const url of local) {+ await expect(readWebPage(url), url).rejects.toThrow("private/internal networks");+ }+ expect(fetchResourceMock).not.toHaveBeenCalled();++ for (const url of [ "http://172.32.0.1/", "http://100.128.0.1/", "http://8.8.8.8/", "http://[2606:4700::1111]/", "http://[fc::1]/", "https://example.com/" ]) {+ serve("ok", "text/plain");+ expect(await readWebPage(url), url).toBe("ok");+ }+ });+it("hands the model a refused address as an error to report", async () => {fetchResourceMock.mockRejectedValueOnce(new Error("URLs pointing to private/internal networks are not allowed"));const tool = createReadWebPageTool();diff --git a/packages/trilium-core/src/services/llm/web_search.ts b/packages/trilium-core/src/services/llm/web_search.tsindex be68926309..f5a951c8ea 100644--- a/packages/trilium-core/src/services/llm/web_search.ts+++ b/packages/trilium-core/src/services/llm/web_search.ts@@ -152,10 +152,16 @@ export function createReadWebPageTool() {** The URL is the model's to choose, and a note or a search result can steer it, so the page is* fetched with `fetchResource()`: the policy for addresses that arrive in content, which refuses- * private networks, vets each redirect and caps the body.+ * private networks, vets each redirect and caps the body. The mobile app's `fetchResource()` has no+ * resolver to make that check, so {@link isLocalHost} refuses the hosts it can recognize by name+ * first.*/export async function readWebPage(url: string): Promise<string> {- const response = await request.fetchResource(validateFetchableUrl(url).toString(), {+ const parsed = validateFetchableUrl(url);+ if (isLocalHost(parsed.hostname)) {+ throw new Error("URLs pointing to private/internal networks are not allowed");+ }+ const response = await request.fetchResource(parsed.toString(), {maxBytes: MAX_PAGE_BYTES,headers: { "Accept": "text/html, application/xhtml+xml, text/plain;q=0.9" }});@@ -176,6 +182,33 @@ export async function readWebPage(url: string): Promise<string> {: content;}+/** Name suffixes that only resolve on a local network. */+const LOCAL_NAME = /(^|\.)(localhost|local|lan|internal|home\.arpa)$/;+/** IPv4 ranges a public page is never served from: this host, private, CGNAT, link-local, multicast and reserved. */+const LOCAL_IPV4 = /^(0|10|127|169\.254|172\.(1[6-9]|2\d|3[01])|192\.168|100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])|22[4-9]|2[3-5]\d)\./;++/**+ * Whether `hostname`, as `URL.hostname` serializes it, names this machine or a private network: a+ * single-label or local-only name, or an address in a non-public range. A name that resolves to a+ * private address passes, since no resolver is available on every runtime.+ */+function isLocalHost(hostname: string): boolean {+ const host = hostname.replace(/\.$/, "").toLowerCase();+ if (host.startsWith("[")) {+ const ipv6 = host.slice(1, -1);+ const mapped = /^::ffff:([\da-f]{1,4}):([\da-f]{1,4})$/.exec(ipv6);+ if (mapped) {+ const [ high, low ] = [ parseInt(mapped[1], 16), parseInt(mapped[2], 16) ];+ return isLocalHost(`${high >> 8}.${high & 0xff}.${low >> 8}.${low & 0xff}`);+ }+ return ipv6 === "::" || ipv6 === "::1" || /^(f[cd][\da-f]{2}|fe[89ab][\da-f]|ff[\da-f]{2}):/.test(ipv6);+ }+ if (/^\d+\.\d+\.\d+\.\d+$/.test(host)) {+ return LOCAL_IPV4.test(host);+ }+ return !host.includes(".") || LOCAL_NAME.test(host);+}+/** The content of a page as Markdown, headed by its title: its `main` or `article` where it has one. */function htmlToMarkdown(html: string): string {const root = parseHtml(html);