Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(fetch): don't time out (for smart scrape)
Gergő Móricz13bd50ad2bc9
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex d6bd51579..c64e79532 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -4,6 +4,7 @@ import * as Sentry from "@sentry/node";import { MockState, saveMock } from "./mock";import { TimeoutSignal } from "../../../controllers/v1/types";import { fireEngineURL } from "../engines/fire-engine/scrape";+import { fetch, RequestInit, Response, FormData, Agent } from "undici";export type RobustFetchParams<Schema extends z.Schema<any>> = {url: string;@@ -78,6 +79,10 @@ export async function robustFetch<...(headers !== undefined ? headers : {}),},signal: abort,+ dispatcher: new Agent({+ headersTimeout: 0,+ bodyTimeout: 0,+ }),...(body instanceof FormData? {body,