Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

fix: fix SSRF handling with redirects in crawler worker (#2759)

Mohamed Bassemc1547297ba7c
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts
index 6eab120c..e71eaff4 100644
--- a/apps/workers/workers/crawlerWorker.ts
+++ b/apps/workers/workers/crawlerWorker.ts
@@ -27,6 +27,7 @@ import {
Browser,
BrowserContext,
BrowserContextOptions,
+ CDPSession,
Page,
} from "playwright";
import { chromium } from "playwright-extra";
@@ -201,6 +202,13 @@ const activeContexts = new Map<
const CONTEXT_CLOSE_TIMEOUT_MS = 10_000;
const PAGE_CLOSE_TIMEOUT_MS = 5_000;
+function getHeaderValue(
+ headers: { name: string; value: string }[] | undefined,
+ name: string,
+): string | undefined {
+ return headers?.find((header) => header.name.toLowerCase() === name)?.value;
+}
+
/**
* Reaps browser contexts that have been open longer than the max job timeout.
* This is a safety net for cases where context.close() hangs or is never called.
@@ -612,6 +620,83 @@ async function crawlPage(
async () => {
// Create a new page in the context
const nextPage = await context.newPage();
+ let cdpSession: CDPSession | undefined;
+
+ try {
+ cdpSession = await context.newCDPSession(nextPage);
+ const continuePausedRequest = async (requestId: string) => {
+ await cdpSession
+ ?.send("Fetch.continueRequest", { requestId })
+ .catch(() => {
+ // Ignore errors — the request may have been canceled.
+ });
+ };
+ const failPausedRequest = async (requestId: string) => {
+ await cdpSession
+ ?.send("Fetch.failRequest", {
+ requestId,
+ errorReason: "BlockedByClient",
+ })
+ .catch(() => {
+ // Ignore errors — the request may have been canceled.
+ });
+ };
+ cdpSession.on("Fetch.requestPaused", async (event) => {
+ try {
+ const status = event.responseStatusCode;
+ if (!status || status < 300 || status >= 400) {
+ await continuePausedRequest(event.requestId);
+ return;
+ }
+
+ const location = getHeaderValue(
+ event.responseHeaders,
+ "location",
+ );
+ if (!location) {
+ await continuePausedRequest(event.requestId);
+ return;
+ }
+
+ const redirectUrl = new URL(
+ location,
+ event.request.url,
+ ).toString();
+ const redirectIsRunningInProxyContext =
+ proxyConfig !== undefined &&
+ !matchesNoProxy(
+ redirectUrl,
+ proxyConfig.bypass?.split(",") ?? [],
+ );
+ const validation = await validateUrl(
+ redirectUrl,
+ redirectIsRunningInProxyContext,
+ );
+
+ if (validation.ok) {
+ await continuePausedRequest(event.requestId);
+ return;
+ }
+
+ logger.warn(
+ `[Crawler][${jobId}] Blocking redirect to disallowed URL "${redirectUrl}": ${validation.reason}`,
+ );
+ await failPausedRequest(event.requestId);
+ } catch (e) {
+ logger.warn(
+ `[Crawler][${jobId}] Blocking redirect after redirect guard failed: ${e}`,
+ );
+ await failPausedRequest(event.requestId);
+ }
+ });
+ await cdpSession.send("Fetch.enable", {
+ patterns: [{ urlPattern: "*", requestStage: "Response" }],
+ });
+ } catch (e) {
+ logger.warn(
+ `[Crawler][${jobId}] Failed to install redirect guard: ${e}`,
+ );
+ }
// Apply ad blocking
if (globalBlocker) {
@@ -678,6 +763,9 @@ async function crawlPage(
abortSignal.addEventListener(
"abort",
() => {
+ cdpSession?.detach().catch(() => {
+ // Ignore errors — the session may already be detached.
+ });
nextPage.unrouteAll({ behavior: "ignoreErrors" }).catch(() => {
// Ignore errors — the page may already be closed.
});
diff --git a/packages/e2e_tests/docker-compose.yml b/packages/e2e_tests/docker-compose.yml
index ea47c3f9..b8ed6264 100644
--- a/packages/e2e_tests/docker-compose.yml
+++ b/packages/e2e_tests/docker-compose.yml
@@ -40,6 +40,7 @@ services:
restart: unless-stopped
volumes:
- ./setup/html:/usr/share/nginx/html
+ - ./setup/nginx/default.conf:/etc/nginx/conf.d/default.conf:ro
minio:
image: minio/minio:latest
restart: unless-stopped
diff --git a/packages/e2e_tests/setup/nginx/default.conf b/packages/e2e_tests/setup/nginx/default.conf
new file mode 100644
index 00000000..a52f5858
--- /dev/null
+++ b/packages/e2e_tests/setup/nginx/default.conf
@@ -0,0 +1,13 @@
+server {
+ listen 80;
+ server_name _;
+ root /usr/share/nginx/html;
+
+ location = /redirect-to-loopback {
+ return 302 http://127.0.0.1:80/hello.html;
+ }
+
+ location / {
+ try_files $uri =404;
+ }
+}
diff --git a/packages/e2e_tests/tests/workers/crawler.test.ts b/packages/e2e_tests/tests/workers/crawler.test.ts
index 1bdef024..5392157d 100644
--- a/packages/e2e_tests/tests/workers/crawler.test.ts
+++ b/packages/e2e_tests/tests/workers/crawler.test.ts
@@ -97,6 +97,52 @@ describe("Crawler Tests", () => {
).toBeDefined();
});
+ it("should fail crawling a disallowed non-redirect URL", async () => {
+ let { data: bookmark } = await client.POST("/bookmarks", {
+ body: {
+ type: "link",
+ url: "http://127.0.0.1:80/hello.html",
+ },
+ });
+ assert(bookmark);
+
+ await waitUntil(async () => {
+ const data = await getBookmark(bookmark!.id);
+ assert(data);
+ assert(data.content.type === "link");
+ return data.content.crawlStatus === "failure";
+ }, "Disallowed non-redirect bookmark crawl fails");
+
+ bookmark = await getBookmark(bookmark.id);
+ assert(bookmark && bookmark.content.type === "link");
+ expect(bookmark.content.crawlStatus).toBe("failure");
+ expect(bookmark.content.crawledAt).toBeNull();
+ expect(bookmark.content.htmlContent).toBeNull();
+ });
+
+ it("should fail crawling a redirect to a disallowed URL", async () => {
+ let { data: bookmark } = await client.POST("/bookmarks", {
+ body: {
+ type: "link",
+ url: "http://nginx:80/redirect-to-loopback",
+ },
+ });
+ assert(bookmark);
+
+ await waitUntil(async () => {
+ const data = await getBookmark(bookmark!.id);
+ assert(data);
+ assert(data.content.type === "link");
+ return data.content.crawlStatus === "failure";
+ }, "Disallowed redirect bookmark crawl fails");
+
+ bookmark = await getBookmark(bookmark.id);
+ assert(bookmark && bookmark.content.type === "link");
+ expect(bookmark.content.crawlStatus).toBe("failure");
+ expect(bookmark.content.crawledAt).toBeNull();
+ expect(bookmark.content.htmlContent).toBeNull();
+ });
+
it("image lings jobs be converted into images", async () => {
let { data: bookmark } = await client.POST("/bookmarks", {
body: {