A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
fix: use safe favicon resolution in metascraper (#2763)
Mohamed Bassem3dc321e7d49a
diff --git a/apps/workers/metascraper-plugins/metascraper-safe-favicon.test.ts b/apps/workers/metascraper-plugins/metascraper-safe-favicon.test.tsnew file mode 100644index 00000000..3af1713a--- /dev/null+++ b/apps/workers/metascraper-plugins/metascraper-safe-favicon.test.ts@@ -0,0 +1,34 @@+import metascraper from "metascraper";+import { describe, expect, test, vi } from "vitest";++vi.mock("network", () => ({+ getRandomProxy: vi.fn((proxies: string[]) => proxies[0]),+}));++vi.mock("@karakeep/shared/config", () => ({+ default: {+ proxy: {},+ },+}));++import metascraperSafeFavicon from "./metascraper-safe-favicon";++describe("metascraperSafeFavicon", () => {+ test("returns favicon candidates discovered by the upstream plugin without fetching them", async () => {+ const parser = metascraper([metascraperSafeFavicon()]);+ const meta = await parser({+ url: "https://example.com/articles/one",+ html: `+ <html>+ <head>+ <link rel="icon" href="/icon-64.png" sizes="64x64">+ <link rel="apple-touch-icon" href="/icon-256.png" sizes="256x256">+ </head>+ </html>+ `,+ validateUrl: false,+ });++ expect(meta.logo).toBe("https://example.com/icon-256.png");+ });+});diff --git a/apps/workers/metascraper-plugins/metascraper-safe-favicon.ts b/apps/workers/metascraper-plugins/metascraper-safe-favicon.tsnew file mode 100644index 00000000..6e362a78--- /dev/null+++ b/apps/workers/metascraper-plugins/metascraper-safe-favicon.ts@@ -0,0 +1,53 @@+import { HttpProxyAgent } from "http-proxy-agent";+import { HttpsProxyAgent } from "https-proxy-agent";+import metascraperLogo from "metascraper-logo-favicon";+import { getRandomProxy } from "network";++import serverConfig from "@karakeep/shared/config";++type FaviconResolver = NonNullable<+ Parameters<typeof metascraperLogo>[0]+>["resolveFaviconUrl"];++interface FaviconResolution {+ url: string;+}++async function resolveSafeFaviconUrl(+ faviconUrl: string,+): Promise<FaviconResolution | undefined> {+ let url: URL;+ try {+ url = new URL(faviconUrl);+ } catch {+ return undefined;+ }++ if (url.protocol !== "http:" && url.protocol !== "https:") {+ return undefined;+ }++ return { url: url.toString() };+}++const metascraperSafeFavicon = () =>+ metascraperLogo({+ gotOpts: {+ agent: {+ http: serverConfig.proxy.httpProxy+ ? new HttpProxyAgent(getRandomProxy(serverConfig.proxy.httpProxy))+ : undefined,+ https: serverConfig.proxy.httpsProxy+ ? new HttpsProxyAgent(getRandomProxy(serverConfig.proxy.httpsProxy))+ : undefined,+ },+ },+ // Do not use the upstream default resolver for page-provided favicon URLs:+ // it checks reachability with reachable-url/got. This resolver only returns+ // normalized http(s) URLs, so attacker-controlled favicon URLs from the HTML+ // are never fetched from the worker.+ resolveFaviconUrl: resolveSafeFaviconUrl as FaviconResolver,+ google: true,+ });++export default metascraperSafeFavicon;diff --git a/apps/workers/scripts/parseHtmlSubprocess.ts b/apps/workers/scripts/parseHtmlSubprocess.tsindex 25413454..e3e15ad1 100644--- a/apps/workers/scripts/parseHtmlSubprocess.ts+++ b/apps/workers/scripts/parseHtmlSubprocess.ts@@ -9,7 +9,6 @@ import metascraperAuthor from "metascraper-author";import metascraperDate from "metascraper-date";import metascraperDescription from "metascraper-description";import metascraperImage from "metascraper-image";-import metascraperLogo from "metascraper-logo-favicon";import metascraperPublisher from "metascraper-publisher";import metascraperTitle from "metascraper-title";import metascraperUrl from "metascraper-url";@@ -23,6 +22,7 @@ import logger from "@karakeep/shared/logger";import metascraperAmazonImproved from "../metascraper-plugins/metascraper-amazon-improved";import metascraperReddit from "../metascraper-plugins/metascraper-reddit";+import metascraperSafeFavicon from "../metascraper-plugins/metascraper-safe-favicon";import {parseSubprocessErrorSchema,parseSubprocessInputSchema,@@ -59,18 +59,7 @@ const metascraperParser = metascraper([metascraperDescription(),metascraperX(),metascraperImage(),- metascraperLogo({- gotOpts: {- agent: {- http: serverConfig.proxy.httpProxy- ? new HttpProxyAgent(getRandomProxy(serverConfig.proxy.httpProxy))- : undefined,- https: serverConfig.proxy.httpsProxy- ? new HttpsProxyAgent(getRandomProxy(serverConfig.proxy.httpsProxy))- : undefined,- },- },- }),+ metascraperSafeFavicon(),metascraperUrl(),]);