Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/engines): better timeouts
Gergő Móricz0f3a27bf2760
diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.tsindex af6f57c03..168d9b8f8 100644--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts@@ -5,8 +5,9 @@ import { specialtyScrapeCheck } from "../utils/specialtyHandler";export async function scrapeURLWithFetch(meta: Meta,+ timeToRun: number | undefined): Promise<EngineScrapeResult> {- const timeout = 20000;+ const timeout = timeToRun ?? 300000;const response = await Promise.race([fetch(meta.url, {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex b84b1e900..ef0b41fc0 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -18,8 +18,6 @@ import * as Sentry from "@sentry/node";import { Action } from "../../../../lib/entities";import { specialtyScrapeCheck } from "../utils/specialtyHandler";-export const defaultTimeout = 10000;-// This function does not take `Meta` on purpose. It may not access any// meta values to construct the request -- that must be done by the// `scrapeURLWithFireEngine*` functions.@@ -31,7 +29,7 @@ async function performFireEngineScrape<>(logger: Logger,request: FireEngineScrapeRequestCommon & Engine,- timeout = defaultTimeout,+ timeout: number,): Promise<FireEngineCheckStatusSuccess> {const scrape = await fireEngineScrape(logger.child({ method: "fireEngineScrape" }),@@ -51,11 +49,7 @@ async function performFireEngineScrape<});}- const userParam = request.timeout ?? 0;- // Use 70% of the user-provided timeout as the timeout for fire-engine check status- const fireEngineTimeout = timeout + Math.round(userParam * 0.7);- const fullTimeout = Math.max(fireEngineTimeout, timeout);- if (Date.now() - startTime > fullTimeout) {+ if (Date.now() - startTime > timeout) {logger.info("Fire-engine was unable to scrape the page before timing out.",{ errors, timeout },@@ -98,6 +92,7 @@ async function performFireEngineScrape<export async function scrapeURLWithFireEngineChromeCDP(meta: Meta,+ timeToRun: number | undefined,): Promise<EngineScrapeResult> {const actions: Action[] = [// Transform waitFor option into an action (unsupported by chrome-cdp)@@ -125,7 +120,7 @@ export async function scrapeURLWithFireEngineChromeCDP(...(meta.options.actions ?? []),];- const timeout = (meta.options.timeout === undefined ? 300000 : Math.round(meta.options.timeout / 3));+ const timeout = timeToRun ?? 300000;const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestChromeCDP = {@@ -212,8 +207,9 @@ export async function scrapeURLWithFireEngineChromeCDP(export async function scrapeURLWithFireEnginePlaywright(meta: Meta,+ timeToRun: number | undefined,): Promise<EngineScrapeResult> {- const timeout = meta.options.timeout === undefined ? 300000 : Math.round(meta.options.timeout / 3);+ const timeout = timeToRun ?? 300000;const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestPlaywright = {@@ -271,8 +267,9 @@ export async function scrapeURLWithFireEnginePlaywright(export async function scrapeURLWithFireEngineTLSClient(meta: Meta,+ timeToRun: number | undefined,): Promise<EngineScrapeResult> {- const timeout = meta.options.timeout === undefined ? 30000 : Math.round(meta.options.timeout / 3);+ const timeout = timeToRun ?? 30000;const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestTLSClient = {diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 01ac0be95..14f263f30 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -105,7 +105,7 @@ export type EngineScrapeResult = {};const engineHandlers: {- [E in Engine]: (meta: Meta) => Promise<EngineScrapeResult>;+ [E in Engine]: (meta: Meta, timeToRun: number | undefined) => Promise<EngineScrapeResult>;} = {cache: scrapeCache,"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,@@ -372,6 +372,7 @@ export function buildFallbackList(meta: Meta): {export async function scrapeURLWithEngine(meta: Meta,engine: Engine,+ timeToRun: number | undefined): Promise<EngineScrapeResult> {const fn = engineHandlers[engine];const logger = meta.logger.child({@@ -383,5 +384,5 @@ export async function scrapeURLWithEngine(logger,};- return await fn(_meta);+ return await fn(_meta, timeToRun);}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 341a4f1a7..24d5f002a 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -15,6 +15,7 @@ type PDFProcessorResult = { html: string; markdown?: string };async function scrapePDFWithLlamaParse(meta: Meta,tempFilePath: string,+ timeToRun: number | undefined,): Promise<PDFProcessorResult> {meta.logger.debug("Processing PDF document with LlamaIndex", {tempFilePath,@@ -63,8 +64,9 @@ async function scrapePDFWithLlamaParse(// TODO: timeout, retriesconst startedAt = Date.now();+ const timeout = timeToRun ?? 300000;- while (Date.now() <= startedAt + (meta.options.timeout ?? 300000)) {+ while (Date.now() <= startedAt + timeout) {try {const result = await robustFetch({url: `https://api.cloud.llamaindex.ai/api/parsing/job/${jobId}/result/markdown`,@@ -122,7 +124,7 @@ async function scrapePDFWithParsePDF(};}-export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {+export async function scrapePDF(meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> {if (!meta.options.parsePDF) {const file = await fetchFileToBuffer(meta.url);const content = file.buffer.toString("base64");@@ -148,6 +150,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {}),},tempFilePath,+ timeToRun,);} catch (error) {if (error instanceof Error && error.message === "LlamaParse timed out") {diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.tsindex c92b1d904..edcd50c02 100644--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts@@ -6,8 +6,9 @@ import { robustFetch } from "../../lib/fetch";export async function scrapeURLWithPlaywright(meta: Meta,+ timeToRun: number | undefined,): Promise<EngineScrapeResult> {- const timeout = 20000 + meta.options.waitFor;+ const timeout = (timeToRun ?? 300000) + meta.options.waitFor;const response = await Promise.race([await robustFetch({@@ -30,7 +31,7 @@ export async function scrapeURLWithPlaywright(}),}),(async () => {- await new Promise((resolve) => setTimeout(() => resolve(null), 20000));+ await new Promise((resolve) => setTimeout(() => resolve(null), timeout));throw new TimeoutError("Playwright was unable to scrape the page before timing out",{ cause: { timeout } },diff --git a/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts b/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.tsindex 50ac502b0..db702a44f 100644--- a/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/scrapingbee/index.ts@@ -9,16 +9,17 @@ const client = new ScrapingBeeClient(process.env.SCRAPING_BEE_API_KEY!);export function scrapeURLWithScrapingBee(wait_browser: "domcontentloaded" | "networkidle2",-): (meta: Meta) => Promise<EngineScrapeResult> {- return async (meta: Meta): Promise<EngineScrapeResult> => {+): (meta: Meta, timeToRun: number | undefined) => Promise<EngineScrapeResult> {+ return async (meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> => {let response: AxiosResponse<any>;+ const timeout = (timeToRun ?? 300000) + meta.options.waitFor;try {response = await client.get({url: meta.url,params: {- timeout: 15000, // TODO: dynamic timeout based on request timeout+ timeout,wait_browser: wait_browser,- wait: Math.min(meta.options.waitFor, 35000),+ wait: meta.options.waitFor,transparent_status_code: true,json_response: true,screenshot: meta.options.formats.includes("screenshot"),diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex d3b33418a..c0b6d4e52 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -202,11 +202,15 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {const results: EngineResultsTracker = {};let result: EngineScrapeResultWithContext | null = null;+ const timeToRun = meta.options.timeout !== undefined+ ? Math.round(meta.options.timeout / Math.min(fallbackList.length, 3))+ : undefined+for (const { engine, unsupportedFeatures } of fallbackList) {const startedAt = Date.now();try {meta.logger.info("Scraping via " + engine + "...");- const _engineResult = await scrapeURLWithEngine(meta, engine);+ const _engineResult = await scrapeURLWithEngine(meta, engine, timeToRun);if (_engineResult.markdown === undefined) {// Some engines emit Markdown directly._engineResult.markdown = await parseMarkdown(_engineResult.html);