A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
fix: fix SSRF handling with redirects in crawler worker (#2759)
Mohamed Bassemc1547297ba7c
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.tsindex 6eab120c..e71eaff4 100644--- a/apps/workers/workers/crawlerWorker.ts+++ b/apps/workers/workers/crawlerWorker.ts@@ -27,6 +27,7 @@ import {Browser,BrowserContext,BrowserContextOptions,+ CDPSession,Page,} from "playwright";import { chromium } from "playwright-extra";@@ -201,6 +202,13 @@ const activeContexts = new Map<const CONTEXT_CLOSE_TIMEOUT_MS = 10_000;const PAGE_CLOSE_TIMEOUT_MS = 5_000;+function getHeaderValue(+ headers: { name: string; value: string }[] | undefined,+ name: string,+): string | undefined {+ return headers?.find((header) => header.name.toLowerCase() === name)?.value;+}+/*** Reaps browser contexts that have been open longer than the max job timeout.* This is a safety net for cases where context.close() hangs or is never called.@@ -612,6 +620,83 @@ async function crawlPage(async () => {// Create a new page in the contextconst nextPage = await context.newPage();+ let cdpSession: CDPSession | undefined;++ try {+ cdpSession = await context.newCDPSession(nextPage);+ const continuePausedRequest = async (requestId: string) => {+ await cdpSession+ ?.send("Fetch.continueRequest", { requestId })+ .catch(() => {+ // Ignore errors — the request may have been canceled.+ });+ };+ const failPausedRequest = async (requestId: string) => {+ await cdpSession+ ?.send("Fetch.failRequest", {+ requestId,+ errorReason: "BlockedByClient",+ })+ .catch(() => {+ // Ignore errors — the request may have been canceled.+ });+ };+ cdpSession.on("Fetch.requestPaused", async (event) => {+ try {+ const status = event.responseStatusCode;+ if (!status || status < 300 || status >= 400) {+ await continuePausedRequest(event.requestId);+ return;+ }++ const location = getHeaderValue(+ event.responseHeaders,+ "location",+ );+ if (!location) {+ await continuePausedRequest(event.requestId);+ return;+ }++ const redirectUrl = new URL(+ location,+ event.request.url,+ ).toString();+ const redirectIsRunningInProxyContext =+ proxyConfig !== undefined &&+ !matchesNoProxy(+ redirectUrl,+ proxyConfig.bypass?.split(",") ?? [],+ );+ const validation = await validateUrl(+ redirectUrl,+ redirectIsRunningInProxyContext,+ );++ if (validation.ok) {+ await continuePausedRequest(event.requestId);+ return;+ }++ logger.warn(+ `[Crawler][${jobId}] Blocking redirect to disallowed URL "${redirectUrl}": ${validation.reason}`,+ );+ await failPausedRequest(event.requestId);+ } catch (e) {+ logger.warn(+ `[Crawler][${jobId}] Blocking redirect after redirect guard failed: ${e}`,+ );+ await failPausedRequest(event.requestId);+ }+ });+ await cdpSession.send("Fetch.enable", {+ patterns: [{ urlPattern: "*", requestStage: "Response" }],+ });+ } catch (e) {+ logger.warn(+ `[Crawler][${jobId}] Failed to install redirect guard: ${e}`,+ );+ }// Apply ad blockingif (globalBlocker) {@@ -678,6 +763,9 @@ async function crawlPage(abortSignal.addEventListener("abort",() => {+ cdpSession?.detach().catch(() => {+ // Ignore errors — the session may already be detached.+ });nextPage.unrouteAll({ behavior: "ignoreErrors" }).catch(() => {// Ignore errors — the page may already be closed.});diff --git a/packages/e2e_tests/docker-compose.yml b/packages/e2e_tests/docker-compose.ymlindex ea47c3f9..b8ed6264 100644--- a/packages/e2e_tests/docker-compose.yml+++ b/packages/e2e_tests/docker-compose.yml@@ -40,6 +40,7 @@ services:restart: unless-stoppedvolumes:- ./setup/html:/usr/share/nginx/html+ - ./setup/nginx/default.conf:/etc/nginx/conf.d/default.conf:rominio:image: minio/minio:latestrestart: unless-stoppeddiff --git a/packages/e2e_tests/setup/nginx/default.conf b/packages/e2e_tests/setup/nginx/default.confnew file mode 100644index 00000000..a52f5858--- /dev/null+++ b/packages/e2e_tests/setup/nginx/default.conf@@ -0,0 +1,13 @@+server {+ listen 80;+ server_name _;+ root /usr/share/nginx/html;++ location = /redirect-to-loopback {+ return 302 http://127.0.0.1:80/hello.html;+ }++ location / {+ try_files $uri =404;+ }+}diff --git a/packages/e2e_tests/tests/workers/crawler.test.ts b/packages/e2e_tests/tests/workers/crawler.test.tsindex 1bdef024..5392157d 100644--- a/packages/e2e_tests/tests/workers/crawler.test.ts+++ b/packages/e2e_tests/tests/workers/crawler.test.ts@@ -97,6 +97,52 @@ describe("Crawler Tests", () => {).toBeDefined();});+ it("should fail crawling a disallowed non-redirect URL", async () => {+ let { data: bookmark } = await client.POST("/bookmarks", {+ body: {+ type: "link",+ url: "http://127.0.0.1:80/hello.html",+ },+ });+ assert(bookmark);++ await waitUntil(async () => {+ const data = await getBookmark(bookmark!.id);+ assert(data);+ assert(data.content.type === "link");+ return data.content.crawlStatus === "failure";+ }, "Disallowed non-redirect bookmark crawl fails");++ bookmark = await getBookmark(bookmark.id);+ assert(bookmark && bookmark.content.type === "link");+ expect(bookmark.content.crawlStatus).toBe("failure");+ expect(bookmark.content.crawledAt).toBeNull();+ expect(bookmark.content.htmlContent).toBeNull();+ });++ it("should fail crawling a redirect to a disallowed URL", async () => {+ let { data: bookmark } = await client.POST("/bookmarks", {+ body: {+ type: "link",+ url: "http://nginx:80/redirect-to-loopback",+ },+ });+ assert(bookmark);++ await waitUntil(async () => {+ const data = await getBookmark(bookmark!.id);+ assert(data);+ assert(data.content.type === "link");+ return data.content.crawlStatus === "failure";+ }, "Disallowed redirect bookmark crawl fails");++ bookmark = await getBookmark(bookmark.id);+ assert(bookmark && bookmark.content.type === "link");+ expect(bookmark.content.crawlStatus).toBe("failure");+ expect(bookmark.content.crawledAt).toBeNull();+ expect(bookmark.content.htmlContent).toBeNull();+ });+it("image lings jobs be converted into images", async () => {let { data: bookmark } = await client.POST("/bookmarks", {body: {