Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): f-e timeout handling (#1774)
Gergő Móricz4a274529c571
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 03391042b..b19eee756 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -368,7 +368,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {startedAt,finishedAt: Date.now(),};- } else if (error instanceof TimeoutError) {+ } else if (error instanceof TimeoutError || (error instanceof Error && error.name === "TimeoutError")) {meta.logger.info("Engine " + engine + " timed out while scraping.", {error,});@@ -435,10 +435,14 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {}if (result === null) {- throw new NoEnginesLeftError(- fallbackList.map((x) => x.engine),- meta.results,- );+ if (Object.values(meta.results).every(x => x.state === "timeout")) {+ throw new TimeoutSignal();+ } else {+ throw new NoEnginesLeftError(+ fallbackList.map((x) => x.engine),+ meta.results,+ );+ }}let document: Document = {diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex e2612ecea..ba9ec7036 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -112,8 +112,8 @@ export async function robustFetch<: {}),});} catch (error) {- if (error instanceof TimeoutSignal) {- throw error;+ if (error instanceof TimeoutSignal || (error instanceof Error && error.name === "TimeoutError")) {+ throw new TimeoutSignal();} else if (!ignoreFailure) {Sentry.captureException(error);if (tryCount > 1) {