Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrape): bound avgrab metadata/download fetches so a slow call can't time out the scrape (#4153)

Abimael Martell2a9dc0753f6e
diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts b/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts
index 9e70415b4..7c1fa248a 100644
--- a/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts
+++ b/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts
@@ -1,4 +1,67 @@
import { youtubePostprocessor } from "../youtube";
+import { config } from "../../../../config";
+
+describe("youtubePostprocessor /metadata fetch timeout", () => {
+ const originalFetch = global.fetch;
+ const originalUrl = config.AVGRAB_SERVICE_URL;
+
+ afterEach(() => {
+ global.fetch = originalFetch;
+ config.AVGRAB_SERVICE_URL = originalUrl;
+ vi.clearAllMocks();
+ });
+
+ it("passes an abort signal so a slow /metadata call can't hang the scrape", async () => {
+ const spy = vi.fn(async (_url: string, _init?: RequestInit) => ({
+ ok: true,
+ json: async () => ({ uploaded_by: {}, transcript: "hi", title: "t" }),
+ }));
+ global.fetch = spy as any;
+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";
+
+ const logger = { warn: vi.fn(), info: vi.fn(), debug: vi.fn() };
+ const meta: any = {
+ logger: { ...logger, child: () => logger },
+ url: "https://www.youtube.com/watch?v=H4fUJQCIV5E",
+ options: { lockdown: false, location: undefined },
+ };
+ // The mocked response fails later shape validation; we only assert the
+ // outgoing request carried a timeout signal.
+ await youtubePostprocessor
+ .run(meta, {
+ url: "https://www.youtube.com/watch?v=H4fUJQCIV5E",
+ markdown: "x",
+ } as any)
+ .catch(() => {});
+
+ const call = spy.mock.calls.find(([u]) => String(u).endsWith("/metadata"));
+ expect(call?.[1]?.signal).toBeInstanceOf(AbortSignal);
+ });
+
+ it("degrades gracefully: a rejected /metadata fetch returns the page unchanged", async () => {
+ global.fetch = vi.fn(async () => {
+ throw Object.assign(new Error("aborted"), { name: "TimeoutError" });
+ }) as any;
+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";
+
+ const logger = { warn: vi.fn(), info: vi.fn(), debug: vi.fn() };
+ const engineResult: any = {
+ url: "https://www.youtube.com/watch?v=H4fUJQCIV5E",
+ markdown: "original page",
+ };
+ const meta: any = {
+ logger: { ...logger, child: () => logger },
+ url: engineResult.url,
+ options: { lockdown: false, location: undefined },
+ };
+ // The postprocessor itself throws; the pipeline's try/catch is what
+ // degrades. Here we assert getYouTubeMetadata rejects (so the caller can
+ // catch), rather than hanging.
+ await expect(
+ youtubePostprocessor.run(meta, engineResult),
+ ).rejects.toBeDefined();
+ });
+});
describe("youtubePostprocessor.shouldRun", () => {
const meta = {} as any;
diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts
index f0733f3fe..7c25279d5 100644
--- a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts
+++ b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts
@@ -4,6 +4,13 @@ import type { Postprocessor } from ".";
import type { EngineScrapeResult } from "../engines";
import { throwIfMediaAccessDenied } from "../error";
+// Transcript enrichment is best-effort: the postprocessor loop swallows its
+// errors and returns the page without a transcript. But an unbounded fetch to
+// avgrab can hang on a slow extraction and silently consume the entire scrape
+// budget, turning "no transcript" into a scrape timeout (SCRAPE_TIMEOUT 500s).
+// A bound lets the graceful degradation actually fire.
+const METADATA_FETCH_TIMEOUT_MS = 45_000;
+
type YouTubeMetadataResponse = {
thumbnail_image: {
url: string;
@@ -112,6 +119,7 @@ async function getYouTubeMetadata(
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(requestBody),
+ signal: AbortSignal.timeout(METADATA_FETCH_TIMEOUT_MS),
});
if (!response.ok) {
diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
index 977d5ab26..a8e9f5bc9 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/audio.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
@@ -4,6 +4,11 @@ import { config } from "../../../config";
import { hasFormatOfType } from "../../../lib/format-utils";
import { AudioUnsupportedUrlError, throwIfMediaAccessDenied } from "../error";
+// Downloads can be large (long videos → hundreds of MB), so this is generous —
+// but an unbounded fetch that hangs would consume the whole scrape budget and
+// surface as an opaque timeout rather than a clean failure.
+const DOWNLOAD_FETCH_TIMEOUT_MS = 240_000;
+
let cachedUrlRegex: RegExp | null = null;
let cacheTimestamp = 0;
const CACHE_TTL_MS = 5 * 60 * 1000;
@@ -72,6 +77,7 @@ export async function fetchAudio(
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(requestBody),
+ signal: AbortSignal.timeout(DOWNLOAD_FETCH_TIMEOUT_MS),
});
if (!response.ok) {
diff --git a/apps/api/src/scraper/scrapeURL/transformers/video.ts b/apps/api/src/scraper/scrapeURL/transformers/video.ts
index b45a06e94..8a2d9d675 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/video.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/video.ts
@@ -4,6 +4,10 @@ import { config } from "../../../config";
import { hasFormatOfType } from "../../../lib/format-utils";
import { throwIfMediaAccessDenied } from "../error";
+// Video downloads can be large; generous but bounded so a hung fetch can't
+// silently consume the whole scrape budget.
+const DOWNLOAD_FETCH_TIMEOUT_MS = 240_000;
+
let cachedUrlRegex: RegExp | null = null;
let cacheTimestamp = 0;
const CACHE_TTL_MS = 5 * 60 * 1000;
@@ -171,6 +175,7 @@ async function fetchLegacyVideoIfSupported(meta: Meta, document: Document) {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(requestBody),
+ signal: AbortSignal.timeout(DOWNLOAD_FETCH_TIMEOUT_MS),
});
if (!response.ok) {