Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): f-e timeout handling (#1774)

Gergő Móricz4a274529c571
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 03391042b..b19eee756 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -368,7 +368,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
startedAt,
finishedAt: Date.now(),
};
- } else if (error instanceof TimeoutError) {
+ } else if (error instanceof TimeoutError || (error instanceof Error && error.name === "TimeoutError")) {
meta.logger.info("Engine " + engine + " timed out while scraping.", {
error,
});
@@ -435,10 +435,14 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
}
if (result === null) {
- throw new NoEnginesLeftError(
- fallbackList.map((x) => x.engine),
- meta.results,
- );
+ if (Object.values(meta.results).every(x => x.state === "timeout")) {
+ throw new TimeoutSignal();
+ } else {
+ throw new NoEnginesLeftError(
+ fallbackList.map((x) => x.engine),
+ meta.results,
+ );
+ }
}
let document: Document = {
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index e2612ecea..ba9ec7036 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -112,8 +112,8 @@ export async function robustFetch<
: {}),
});
} catch (error) {
- if (error instanceof TimeoutSignal) {
- throw error;
+ if (error instanceof TimeoutSignal || (error instanceof Error && error.name === "TimeoutError")) {
+ throw new TimeoutSignal();
} else if (!ignoreFailure) {
Sentry.captureException(error);
if (tryCount > 1) {