Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrape): bound avgrab metadata/download fetches so a slow call can't time out the scrape (#4153)
Abimael Martell2a9dc0753f6e
diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts b/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.tsindex 9e70415b4..7c1fa248a 100644--- a/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts+++ b/apps/api/src/scraper/scrapeURL/postprocessors/__tests__/youtube.test.ts@@ -1,4 +1,67 @@import { youtubePostprocessor } from "../youtube";+import { config } from "../../../../config";++describe("youtubePostprocessor /metadata fetch timeout", () => {+ const originalFetch = global.fetch;+ const originalUrl = config.AVGRAB_SERVICE_URL;++ afterEach(() => {+ global.fetch = originalFetch;+ config.AVGRAB_SERVICE_URL = originalUrl;+ vi.clearAllMocks();+ });++ it("passes an abort signal so a slow /metadata call can't hang the scrape", async () => {+ const spy = vi.fn(async (_url: string, _init?: RequestInit) => ({+ ok: true,+ json: async () => ({ uploaded_by: {}, transcript: "hi", title: "t" }),+ }));+ global.fetch = spy as any;+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";++ const logger = { warn: vi.fn(), info: vi.fn(), debug: vi.fn() };+ const meta: any = {+ logger: { ...logger, child: () => logger },+ url: "https://www.youtube.com/watch?v=H4fUJQCIV5E",+ options: { lockdown: false, location: undefined },+ };+ // The mocked response fails later shape validation; we only assert the+ // outgoing request carried a timeout signal.+ await youtubePostprocessor+ .run(meta, {+ url: "https://www.youtube.com/watch?v=H4fUJQCIV5E",+ markdown: "x",+ } as any)+ .catch(() => {});++ const call = spy.mock.calls.find(([u]) => String(u).endsWith("/metadata"));+ expect(call?.[1]?.signal).toBeInstanceOf(AbortSignal);+ });++ it("degrades gracefully: a rejected /metadata fetch returns the page unchanged", async () => {+ global.fetch = vi.fn(async () => {+ throw Object.assign(new Error("aborted"), { name: "TimeoutError" });+ }) as any;+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";++ const logger = { warn: vi.fn(), info: vi.fn(), debug: vi.fn() };+ const engineResult: any = {+ url: "https://www.youtube.com/watch?v=H4fUJQCIV5E",+ markdown: "original page",+ };+ const meta: any = {+ logger: { ...logger, child: () => logger },+ url: engineResult.url,+ options: { lockdown: false, location: undefined },+ };+ // The postprocessor itself throws; the pipeline's try/catch is what+ // degrades. Here we assert getYouTubeMetadata rejects (so the caller can+ // catch), rather than hanging.+ await expect(+ youtubePostprocessor.run(meta, engineResult),+ ).rejects.toBeDefined();+ });+});describe("youtubePostprocessor.shouldRun", () => {const meta = {} as any;diff --git a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.tsindex f0733f3fe..7c25279d5 100644--- a/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts+++ b/apps/api/src/scraper/scrapeURL/postprocessors/youtube.ts@@ -4,6 +4,13 @@ import type { Postprocessor } from ".";import type { EngineScrapeResult } from "../engines";import { throwIfMediaAccessDenied } from "../error";+// Transcript enrichment is best-effort: the postprocessor loop swallows its+// errors and returns the page without a transcript. But an unbounded fetch to+// avgrab can hang on a slow extraction and silently consume the entire scrape+// budget, turning "no transcript" into a scrape timeout (SCRAPE_TIMEOUT 500s).+// A bound lets the graceful degradation actually fire.+const METADATA_FETCH_TIMEOUT_MS = 45_000;+type YouTubeMetadataResponse = {thumbnail_image: {url: string;@@ -112,6 +119,7 @@ async function getYouTubeMetadata(method: "POST",headers: { "Content-Type": "application/json" },body: JSON.stringify(requestBody),+ signal: AbortSignal.timeout(METADATA_FETCH_TIMEOUT_MS),});if (!response.ok) {diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.tsindex 977d5ab26..a8e9f5bc9 100644--- a/apps/api/src/scraper/scrapeURL/transformers/audio.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts@@ -4,6 +4,11 @@ import { config } from "../../../config";import { hasFormatOfType } from "../../../lib/format-utils";import { AudioUnsupportedUrlError, throwIfMediaAccessDenied } from "../error";+// Downloads can be large (long videos → hundreds of MB), so this is generous —+// but an unbounded fetch that hangs would consume the whole scrape budget and+// surface as an opaque timeout rather than a clean failure.+const DOWNLOAD_FETCH_TIMEOUT_MS = 240_000;+let cachedUrlRegex: RegExp | null = null;let cacheTimestamp = 0;const CACHE_TTL_MS = 5 * 60 * 1000;@@ -72,6 +77,7 @@ export async function fetchAudio(method: "POST",headers: { "Content-Type": "application/json" },body: JSON.stringify(requestBody),+ signal: AbortSignal.timeout(DOWNLOAD_FETCH_TIMEOUT_MS),});if (!response.ok) {diff --git a/apps/api/src/scraper/scrapeURL/transformers/video.ts b/apps/api/src/scraper/scrapeURL/transformers/video.tsindex b45a06e94..8a2d9d675 100644--- a/apps/api/src/scraper/scrapeURL/transformers/video.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/video.ts@@ -4,6 +4,10 @@ import { config } from "../../../config";import { hasFormatOfType } from "../../../lib/format-utils";import { throwIfMediaAccessDenied } from "../error";+// Video downloads can be large; generous but bounded so a hung fetch can't+// silently consume the whole scrape budget.+const DOWNLOAD_FETCH_TIMEOUT_MS = 240_000;+let cachedUrlRegex: RegExp | null = null;let cacheTimestamp = 0;const CACHE_TTL_MS = 5 * 60 * 1000;@@ -171,6 +175,7 @@ async function fetchLegacyVideoIfSupported(meta: Meta, document: Document) {method: "POST",headers: { "Content-Type": "application/json" },body: JSON.stringify(requestBody),+ signal: AbortSignal.timeout(DOWNLOAD_FETCH_TIMEOUT_MS),});if (!response.ok) {