A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
fix: harden video worker redirect validation against SSRF (#2760)
Mohamed Bassem43f50e85413a
diff --git a/apps/workers/network.ts b/apps/workers/network.tsindex d22beb58..a5a83299 100644--- a/apps/workers/network.ts+++ b/apps/workers/network.ts@@ -1,4 +1,5 @@import dns from "node:dns/promises";+import { Readable } from "node:stream";import type { HeadersInit, RequestInit, Response } from "node-fetch";import { HttpProxyAgent } from "http-proxy-agent";import { HttpsProxyAgent } from "https-proxy-agent";@@ -335,6 +336,22 @@ function isRedirectResponse(response: Response): boolean {);}+function closeResponseBody(response: Response): void {+ const body: unknown = response.body;+ if (!body) {+ return;+ }++ if (body instanceof Readable) {+ body.destroy();+ } else if (+ typeof ReadableStream !== "undefined" &&+ body instanceof ReadableStream+ ) {+ void body.cancel();+ }+}+export type FetchWithProxyOptions = Omit<RequestInit & {maxRedirects?: number;@@ -467,3 +484,69 @@ export const fetchWithProxy = async (redirectsRemaining -= 1;}};++export async function resolveValidatedRedirectUrl(+ url: string,+ options: Pick<+ FetchWithProxyOptions,+ "headers" | "maxRedirects" | "signal"+ > = {},+ runProxy?: RunProxyConfig,+): Promise<URL> {+ const { maxRedirects, baseHeaders, baseOptions } = prepareFetchOptions({+ ...options,+ method: "GET",+ });++ let redirectsRemaining = maxRedirects;+ let currentUrl = url;++ while (true) {+ const signal = options.signal+ ? AbortSignal.any([+ AbortSignal.timeout(5000),+ options.signal as globalThis.AbortSignal,+ ])+ : AbortSignal.timeout(5000);+ const agent = getProxyAgent(currentUrl, runProxy);+ const validation = await validateUrl(currentUrl, !!agent);+ if (!validation.ok) {+ throw new Error(validation.reason);+ }++ const requestUrl = validation.url;+ currentUrl = requestUrl.toString();++ const response = await fetch(+ currentUrl,+ buildFetchOptions({+ method: "GET",+ headers: baseHeaders,+ agent,+ baseOptions: {+ ...baseOptions,+ signal: signal as RequestInit["signal"],+ },+ }),+ );++ if (!isRedirectResponse(response)) {+ closeResponseBody(response);+ return requestUrl;+ }++ closeResponseBody(response);++ const locationHeader = response.headers.get("location");+ if (!locationHeader) {+ return requestUrl;+ }++ if (redirectsRemaining <= 0) {+ throw new Error(`Too many redirects while resolving ${url}`);+ }++ currentUrl = new URL(locationHeader, currentUrl).toString();+ redirectsRemaining -= 1;+ }+}diff --git a/apps/workers/workers/videoWorker.ts b/apps/workers/workers/videoWorker.tsindex dd937a82..c7fa138f 100644--- a/apps/workers/workers/videoWorker.ts+++ b/apps/workers/workers/videoWorker.ts@@ -3,7 +3,11 @@ import * as os from "os";import path from "path";import { execa } from "execa";import { workerStatsCounter } from "metrics";-import { getProxyAgent, validateUrl } from "network";+import {+ getProxyAgent,+ resolveValidatedRedirectUrl,+ selectRunProxies,+} from "network";import { withWorkerEventLog, withWorkerTracing } from "workerTracing";import { db } from "@karakeep/db";@@ -76,6 +80,9 @@ function prepareYtDlpArguments(proxy: string | undefined,assetPath: string,) {+ // yt-dlp performs its own HTTP requests and can follow redirects that this+ // process cannot validate. Full SSRF protection depends on an egress proxy or+ // network policy that blocks internal/private targets.const ytDlpArguments = [url];if (serverConfig.crawler.maxVideoDownloadSize > 0) {ytDlpArguments.push(@@ -111,20 +118,29 @@ async function runWorker(job: DequeuedJob<ZVideoRequest>) {return;}- const proxy = getProxyAgent(url);- const validation = await validateUrl(url, !!proxy);- if (!validation.ok) {+ const runProxy = selectRunProxies();+ let normalizedUrl: string;+ try {+ const resolvedUrl = await resolveValidatedRedirectUrl(+ url,+ { signal: job.abortSignal },+ runProxy,+ );+ normalizedUrl = resolvedUrl.toString();+ } catch (error) {logger.warn(- `[VideoCrawler][${jobId}] Skipping video download to disallowed URL "${url}": ${validation.reason}`,+ `[VideoCrawler][${jobId}] Skipping video download for "${url}": ${+ error instanceof Error ? error.message : String(error)+ }`,);return;}- const normalizedUrl = validation.url.toString();const videoAssetId = newAssetId();let assetPath = `${TMP_FOLDER}/${videoAssetId}`;await fs.promises.mkdir(TMP_FOLDER, { recursive: true });+ const proxy = getProxyAgent(normalizedUrl, runProxy);const ytDlpArguments = prepareYtDlpArguments(normalizedUrl,proxy?.proxy.toString(),diff --git a/packages/e2e_tests/docker-compose.yml b/packages/e2e_tests/docker-compose.ymlindex b8ed6264..20283a82 100644--- a/packages/e2e_tests/docker-compose.yml+++ b/packages/e2e_tests/docker-compose.yml@@ -16,6 +16,8 @@ services:BROWSER_WEB_URL: http://chrome:9222CRAWLER_NUM_WORKERS: 6CRAWLER_ALLOWED_INTERNAL_HOSTNAMES: nginx+ CRAWLER_VIDEO_DOWNLOAD: "true"+ CRAWLER_VIDEO_DOWNLOAD_MAX_SIZE: -1OPENAI_API_KEY: aimock-test-keyOPENAI_BASE_URL: http://aimock:4010/v1INFERENCE_ENABLE_AUTO_SUMMARIZATION: "true"diff --git a/packages/e2e_tests/setup/html/video.html b/packages/e2e_tests/setup/html/video.htmlnew file mode 100644index 00000000..6ce5a8ac--- /dev/null+++ b/packages/e2e_tests/setup/html/video.html@@ -0,0 +1,9 @@+<!doctype html>+<html>+ <head>+ <title>Video fixture</title>+ </head>+ <body>+ <video controls src="/video.mp4"></video>+ </body>+</html>diff --git a/packages/e2e_tests/setup/html/video.mp4 b/packages/e2e_tests/setup/html/video.mp4new file mode 100644index 00000000..60398e89--- /dev/null+++ b/packages/e2e_tests/setup/html/video.mp4@@ -0,0 +1 @@+video fixture for yt-dlp generic extractordiff --git a/packages/e2e_tests/tests/workers/video.test.ts b/packages/e2e_tests/tests/workers/video.test.tsnew file mode 100644index 00000000..21427682--- /dev/null+++ b/packages/e2e_tests/tests/workers/video.test.ts@@ -0,0 +1,68 @@+import { assert, beforeEach, describe, expect, inject, it } from "vitest";++import { createKarakeepClient } from "@karakeep/sdk";++import { createTestUser } from "../../utils/api";+import { waitUntil } from "../../utils/general";++describe("Video Worker Tests", () => {+ const port = inject("karakeepPort");++ if (!port) {+ throw new Error("Missing required environment variables");+ }++ let client: ReturnType<typeof createKarakeepClient>;+ let apiKey: string;++ async function getBookmark(bookmarkId: string) {+ const { data } = await client.GET(`/bookmarks/{bookmarkId}`, {+ params: {+ path: {+ bookmarkId,+ },+ query: {+ includeContent: true,+ },+ },+ });+ return data;+ }++ beforeEach(async () => {+ apiKey = await createTestUser();+ client = createKarakeepClient({+ baseUrl: `http://localhost:${port}/api/v1/`,+ headers: {+ "Content-Type": "application/json",+ authorization: `Bearer ${apiKey}`,+ },+ });+ });++ it("downloads a video attachment using yt-dlp", async () => {+ let { data: bookmark } = await client.POST("/bookmarks", {+ body: {+ type: "link",+ url: "http://nginx:80/video.html",+ },+ });+ assert(bookmark);++ await waitUntil(+ async () => {+ const data = await getBookmark(bookmark!.id);+ assert(data);+ assert(data.content.type === "link");+ return !!data.content.videoAssetId;+ },+ "Video attachment is downloaded",+ 120000,+ );++ bookmark = await getBookmark(bookmark.id);+ assert(bookmark && bookmark.content.type === "link");+ expect(bookmark.content.videoAssetId).toBeDefined();+ expect(bookmark.assets.find((a) => a.assetType === "video")).toBeDefined();+ });+});