Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

fix: harden video worker redirect validation against SSRF (#2760)

Mohamed Bassem43f50e85413a
diff --git a/apps/workers/network.ts b/apps/workers/network.ts
index d22beb58..a5a83299 100644
--- a/apps/workers/network.ts
+++ b/apps/workers/network.ts
@@ -1,4 +1,5 @@
import dns from "node:dns/promises";
+import { Readable } from "node:stream";
import type { HeadersInit, RequestInit, Response } from "node-fetch";
import { HttpProxyAgent } from "http-proxy-agent";
import { HttpsProxyAgent } from "https-proxy-agent";
@@ -335,6 +336,22 @@ function isRedirectResponse(response: Response): boolean {
);
}
+function closeResponseBody(response: Response): void {
+ const body: unknown = response.body;
+ if (!body) {
+ return;
+ }
+
+ if (body instanceof Readable) {
+ body.destroy();
+ } else if (
+ typeof ReadableStream !== "undefined" &&
+ body instanceof ReadableStream
+ ) {
+ void body.cancel();
+ }
+}
+
export type FetchWithProxyOptions = Omit<
RequestInit & {
maxRedirects?: number;
@@ -467,3 +484,69 @@ export const fetchWithProxy = async (
redirectsRemaining -= 1;
}
};
+
+export async function resolveValidatedRedirectUrl(
+ url: string,
+ options: Pick<
+ FetchWithProxyOptions,
+ "headers" | "maxRedirects" | "signal"
+ > = {},
+ runProxy?: RunProxyConfig,
+): Promise<URL> {
+ const { maxRedirects, baseHeaders, baseOptions } = prepareFetchOptions({
+ ...options,
+ method: "GET",
+ });
+
+ let redirectsRemaining = maxRedirects;
+ let currentUrl = url;
+
+ while (true) {
+ const signal = options.signal
+ ? AbortSignal.any([
+ AbortSignal.timeout(5000),
+ options.signal as globalThis.AbortSignal,
+ ])
+ : AbortSignal.timeout(5000);
+ const agent = getProxyAgent(currentUrl, runProxy);
+ const validation = await validateUrl(currentUrl, !!agent);
+ if (!validation.ok) {
+ throw new Error(validation.reason);
+ }
+
+ const requestUrl = validation.url;
+ currentUrl = requestUrl.toString();
+
+ const response = await fetch(
+ currentUrl,
+ buildFetchOptions({
+ method: "GET",
+ headers: baseHeaders,
+ agent,
+ baseOptions: {
+ ...baseOptions,
+ signal: signal as RequestInit["signal"],
+ },
+ }),
+ );
+
+ if (!isRedirectResponse(response)) {
+ closeResponseBody(response);
+ return requestUrl;
+ }
+
+ closeResponseBody(response);
+
+ const locationHeader = response.headers.get("location");
+ if (!locationHeader) {
+ return requestUrl;
+ }
+
+ if (redirectsRemaining <= 0) {
+ throw new Error(`Too many redirects while resolving ${url}`);
+ }
+
+ currentUrl = new URL(locationHeader, currentUrl).toString();
+ redirectsRemaining -= 1;
+ }
+}
diff --git a/apps/workers/workers/videoWorker.ts b/apps/workers/workers/videoWorker.ts
index dd937a82..c7fa138f 100644
--- a/apps/workers/workers/videoWorker.ts
+++ b/apps/workers/workers/videoWorker.ts
@@ -3,7 +3,11 @@ import * as os from "os";
import path from "path";
import { execa } from "execa";
import { workerStatsCounter } from "metrics";
-import { getProxyAgent, validateUrl } from "network";
+import {
+ getProxyAgent,
+ resolveValidatedRedirectUrl,
+ selectRunProxies,
+} from "network";
import { withWorkerEventLog, withWorkerTracing } from "workerTracing";
import { db } from "@karakeep/db";
@@ -76,6 +80,9 @@ function prepareYtDlpArguments(
proxy: string | undefined,
assetPath: string,
) {
+ // yt-dlp performs its own HTTP requests and can follow redirects that this
+ // process cannot validate. Full SSRF protection depends on an egress proxy or
+ // network policy that blocks internal/private targets.
const ytDlpArguments = [url];
if (serverConfig.crawler.maxVideoDownloadSize > 0) {
ytDlpArguments.push(
@@ -111,20 +118,29 @@ async function runWorker(job: DequeuedJob<ZVideoRequest>) {
return;
}
- const proxy = getProxyAgent(url);
- const validation = await validateUrl(url, !!proxy);
- if (!validation.ok) {
+ const runProxy = selectRunProxies();
+ let normalizedUrl: string;
+ try {
+ const resolvedUrl = await resolveValidatedRedirectUrl(
+ url,
+ { signal: job.abortSignal },
+ runProxy,
+ );
+ normalizedUrl = resolvedUrl.toString();
+ } catch (error) {
logger.warn(
- `[VideoCrawler][${jobId}] Skipping video download to disallowed URL "${url}": ${validation.reason}`,
+ `[VideoCrawler][${jobId}] Skipping video download for "${url}": ${
+ error instanceof Error ? error.message : String(error)
+ }`,
);
return;
}
- const normalizedUrl = validation.url.toString();
const videoAssetId = newAssetId();
let assetPath = `${TMP_FOLDER}/${videoAssetId}`;
await fs.promises.mkdir(TMP_FOLDER, { recursive: true });
+ const proxy = getProxyAgent(normalizedUrl, runProxy);
const ytDlpArguments = prepareYtDlpArguments(
normalizedUrl,
proxy?.proxy.toString(),
diff --git a/packages/e2e_tests/docker-compose.yml b/packages/e2e_tests/docker-compose.yml
index b8ed6264..20283a82 100644
--- a/packages/e2e_tests/docker-compose.yml
+++ b/packages/e2e_tests/docker-compose.yml
@@ -16,6 +16,8 @@ services:
BROWSER_WEB_URL: http://chrome:9222
CRAWLER_NUM_WORKERS: 6
CRAWLER_ALLOWED_INTERNAL_HOSTNAMES: nginx
+ CRAWLER_VIDEO_DOWNLOAD: "true"
+ CRAWLER_VIDEO_DOWNLOAD_MAX_SIZE: -1
OPENAI_API_KEY: aimock-test-key
OPENAI_BASE_URL: http://aimock:4010/v1
INFERENCE_ENABLE_AUTO_SUMMARIZATION: "true"
diff --git a/packages/e2e_tests/setup/html/video.html b/packages/e2e_tests/setup/html/video.html
new file mode 100644
index 00000000..6ce5a8ac
--- /dev/null
+++ b/packages/e2e_tests/setup/html/video.html
@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+ <head>
+ <title>Video fixture</title>
+ </head>
+ <body>
+ <video controls src="/video.mp4"></video>
+ </body>
+</html>
diff --git a/packages/e2e_tests/setup/html/video.mp4 b/packages/e2e_tests/setup/html/video.mp4
new file mode 100644
index 00000000..60398e89
--- /dev/null
+++ b/packages/e2e_tests/setup/html/video.mp4
@@ -0,0 +1 @@
+video fixture for yt-dlp generic extractor
diff --git a/packages/e2e_tests/tests/workers/video.test.ts b/packages/e2e_tests/tests/workers/video.test.ts
new file mode 100644
index 00000000..21427682
--- /dev/null
+++ b/packages/e2e_tests/tests/workers/video.test.ts
@@ -0,0 +1,68 @@
+import { assert, beforeEach, describe, expect, inject, it } from "vitest";
+
+import { createKarakeepClient } from "@karakeep/sdk";
+
+import { createTestUser } from "../../utils/api";
+import { waitUntil } from "../../utils/general";
+
+describe("Video Worker Tests", () => {
+ const port = inject("karakeepPort");
+
+ if (!port) {
+ throw new Error("Missing required environment variables");
+ }
+
+ let client: ReturnType<typeof createKarakeepClient>;
+ let apiKey: string;
+
+ async function getBookmark(bookmarkId: string) {
+ const { data } = await client.GET(`/bookmarks/{bookmarkId}`, {
+ params: {
+ path: {
+ bookmarkId,
+ },
+ query: {
+ includeContent: true,
+ },
+ },
+ });
+ return data;
+ }
+
+ beforeEach(async () => {
+ apiKey = await createTestUser();
+ client = createKarakeepClient({
+ baseUrl: `http://localhost:${port}/api/v1/`,
+ headers: {
+ "Content-Type": "application/json",
+ authorization: `Bearer ${apiKey}`,
+ },
+ });
+ });
+
+ it("downloads a video attachment using yt-dlp", async () => {
+ let { data: bookmark } = await client.POST("/bookmarks", {
+ body: {
+ type: "link",
+ url: "http://nginx:80/video.html",
+ },
+ });
+ assert(bookmark);
+
+ await waitUntil(
+ async () => {
+ const data = await getBookmark(bookmark!.id);
+ assert(data);
+ assert(data.content.type === "link");
+ return !!data.content.videoAssetId;
+ },
+ "Video attachment is downloaded",
+ 120000,
+ );
+
+ bookmark = await getBookmark(bookmark.id);
+ assert(bookmark && bookmark.content.type === "link");
+ expect(bookmark.content.videoAssetId).toBeDefined();
+ expect(bookmark.assets.find((a) => a.assetType === "video")).toBeDefined();
+ });
+});