Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): ask user to increase timeout if there's a DOM.getDocument or queryAXTree error (#1739)
Gergő Móricz400d497fcab5
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 68d107fc4..0b43f3e8e 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -9,7 +9,8 @@ import {SiteError,SSLError,UnsupportedFileError,- DNSResolutionError+ DNSResolutionError,+ FEPageLoadFailed} from "../../error";import { MockState } from "../../lib/mock";import { fireEngineURL } from "./scrape";@@ -202,6 +203,12 @@ export async function fireEngineCheckStatus(throw new UnsupportedFileError("File size exceeds " + status.error.split("File size exceeds ")[1],);+ } else if (+ typeof status.error === "string" &&+ status.error.includes("failed to finish without timing out")+ ) {+ logger.warn("CDP timed out while loading the page", { status, jobId });+ throw new FEPageLoadFailed();} else if (typeof status.error === "string" &&// TODO: improve this laterdiff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex a3c0c1561..66349fbc7 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -21,6 +21,7 @@ import {SSLError,TimeoutError,UnsupportedFileError,+ FEPageLoadFailed,} from "../../error";import * as Sentry from "@sentry/node";import { specialtyScrapeCheck } from "../utils/specialtyHandler";@@ -102,7 +103,8 @@ async function performFireEngineScrape<error instanceof SSLError ||error instanceof DNSResolutionError ||error instanceof ActionError ||- error instanceof UnsupportedFileError+ error instanceof UnsupportedFileError ||+ error instanceof FEPageLoadFailed) {fireEngineDelete(logger.child({diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex df3dd05e0..5afcca69c 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -115,3 +115,9 @@ export class PDFPrefetchFailed extends Error {super("Failed to prefetch PDF that is protected by anti-bot. Please contact [email protected]");}}++export class FEPageLoadFailed extends Error {+ constructor() {+ super("The page failed to load with the specified timeout. Please increase the timeout parameter in your request.");+ }+}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex e0f74ef08..3493c015b 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -26,6 +26,7 @@ import {IndexMissError,DNSResolutionError,PDFPrefetchFailed,+ FEPageLoadFailed,} from "./error";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";@@ -65,6 +66,7 @@ export type Meta = {} | null | undefined; // undefined: no prefetch yet, null: prefetch came back emptycostTracking: CostTracking;winnerEngine?: Engine;+ results: EngineResultsTracker;};function buildFeatureFlags(@@ -193,6 +195,7 @@ async function buildMetaObject(: null,pdfPrefetch: undefined,costTracking,+ results: {},};}@@ -261,7 +264,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {const fallbackList = buildFallbackList(meta);- const results: EngineResultsTracker = {};let result: EngineScrapeResultWithContext | null = null;const timeToRun =@@ -293,7 +295,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {const hasNoPageError = engineResult.error === undefined;const isLikelyProxyError = [403, 429].includes(engineResult.statusCode);- results[engine] = {+ meta.results[engine] = {state: "success",result: engineResult,factors: { isLongEnough, isGoodStatusCode, hasNoPageError, isLikelyProxyError },@@ -327,7 +329,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {meta.logger.warn("Engine " + engine + " could not scrape the page.", {error,});- results[engine] = {+ meta.results[engine] = {state: "error",error: safeguardCircularError(error),unexpected: false,@@ -338,7 +340,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {meta.logger.info("Engine " + engine + " could not find the page in the index.", {error,});- results[engine] = {+ meta.results[engine] = {state: "error",error: safeguardCircularError(error),unexpected: false,@@ -349,7 +351,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {meta.logger.info("Engine " + engine + " timed out while scraping.", {error,});- results[engine] = {+ meta.results[engine] = {state: "timeout",startedAt,finishedAt: Date.now(),@@ -360,14 +362,14 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {) {throw error;} else if (error instanceof LLMRefusalError) {- results[engine] = {+ meta.results[engine] = {state: "error",error: safeguardCircularError(error),unexpected: true,startedAt,finishedAt: Date.now(),};- error.results = results;+ error.results = meta.results;meta.logger.warn("LLM refusal encountered", { error });throw error;} else if (error instanceof SiteError) {@@ -386,13 +388,21 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {throw error;} else if (error instanceof PDFInsufficientTimeError) {throw error;+ } else if (error instanceof FEPageLoadFailed) {+ meta.results[engine] = {+ state: "error",+ error,+ unexpected: false,+ startedAt,+ finishedAt: Date.now(),+ };} else {Sentry.captureException(error);meta.logger.warn("An unexpected error happened while scraping with " + engine + ".",{ error },);- results[engine] = {+ meta.results[engine] = {state: "error",error: safeguardCircularError(error),unexpected: true,@@ -406,7 +416,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {if (result === null) {throw new NoEnginesLeftError(fallbackList.map((x) => x.engine),- results,+ meta.results,);}@@ -423,7 +433,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {numPages: result.result.numPages,contentType: result.result.contentType,proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",- ...((results["index"] || results["index;documents"]) ? (+ ...((meta.results["index"] || meta.results["index;documents"]) ? (result.result.cacheInfo ? {cacheState: "hit",cachedAt: result.result.cacheInfo.created_at.toISOString(),@@ -452,7 +462,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {success: true,document,logs: meta.logs,- engines: results,+ engines: meta.results,};}@@ -524,14 +534,12 @@ export async function scrapeURL(}}} catch (error) {- let results: EngineResultsTracker = {};-- if (error instanceof NoEnginesLeftError) {+ if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {+ throw new FEPageLoadFailed();+ } else if (error instanceof NoEnginesLeftError) {meta.logger.warn("scrapeURL: All scraping engines failed!", { error });- results = error.results;} else if (error instanceof LLMRefusalError) {meta.logger.warn("scrapeURL: LLM refused to extract content", { error });- results = error.results!;} else if (error instanceof Error &&error.message.includes("Invalid schema for response_format")@@ -565,7 +573,7 @@ export async function scrapeURL(success: false,error,logs: meta.logs,- engines: results,+ engines: meta.results,};}}