Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(fetch): don't time out (for smart scrape)

Gergő Móricz13bd50ad2bc9
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index d6bd51579..c64e79532 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -4,6 +4,7 @@ import * as Sentry from "@sentry/node";
import { MockState, saveMock } from "./mock";
import { TimeoutSignal } from "../../../controllers/v1/types";
import { fireEngineURL } from "../engines/fire-engine/scrape";
+import { fetch, RequestInit, Response, FormData, Agent } from "undici";
export type RobustFetchParams<Schema extends z.Schema<any>> = {
url: string;
@@ -78,6 +79,10 @@ export async function robustFetch<
...(headers !== undefined ? headers : {}),
},
signal: abort,
+ dispatcher: new Agent({
+ headersTimeout: 0,
+ bodyTimeout: 0,
+ }),
...(body instanceof FormData
? {
body,