Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): scope fetch state
mogery656bffcc2883
diff --git a/apps/api/src/lib/validateUrl.ts b/apps/api/src/lib/validateUrl.tsindex a73c06f0d..3dcdd7119 100644--- a/apps/api/src/lib/validateUrl.ts+++ b/apps/api/src/lib/validateUrl.ts@@ -200,6 +200,7 @@ export async function resolveRedirects(const targetUrl = protocolIncluded(url) ? url : `http://${url}`;const methods = ["HEAD", "GET"] as const;+ const dispatcher = getSecureDispatcher(false);for (const method of methods) {const signal = abort@@ -210,7 +211,7 @@ export async function resolveRedirects(const response = await undici.fetch(targetUrl, {method,redirect: "follow",- dispatcher: getSecureDispatcher(false),+ dispatcher,signal,});diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex 3b0ad57e5..4ef7ca8ad 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -60,11 +60,8 @@ function attachSecurityCheck(agent: undici.Dispatcher) {});}-// Dispatcher WITH cookie handling (for scraping - needs cookies for auth flows)function makeSecureDispatcher(skipTlsVerification: boolean) {- const baseAgent = createBaseAgent(skipTlsVerification);- const cookieJar = new CookieJar();- const agent = baseAgent.compose(cookie({ jar: cookieJar }));+ const agent = createBaseAgent(skipTlsVerification);attachSecurityCheck(agent);return agent;}@@ -82,8 +79,13 @@ const secureDispatcherNoCookies = makeSecureDispatcherNoCookies(false);const secureDispatcherNoCookiesSkipTlsVerification =makeSecureDispatcherNoCookies(true);-export const getSecureDispatcher = (skipTlsVerification: boolean = false) =>- skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher;+export const getSecureDispatcher = (skipTlsVerification: boolean = false) => {+ const dispatcher = skipTlsVerification+ ? secureDispatcherSkipTlsVerification+ : secureDispatcher;++ return dispatcher.compose(cookie({ jar: new CookieJar() }));+};// Use this for webhook delivery to avoid sending empty cookie headersexport const getSecureDispatcherNoCookies = (diff --git a/apps/api/src/search/v2/ddgsearch.ts b/apps/api/src/search/v2/ddgsearch.tsindex 9838423ac..9b2571f4f 100644--- a/apps/api/src/search/v2/ddgsearch.ts+++ b/apps/api/src/search/v2/ddgsearch.ts@@ -130,6 +130,7 @@ export async function ddgSearch(const results: WebSearchResult[] = [];const seenUrls = new Set<string>();+ const dispatcher = getSecureDispatcher(false);let isFirstPage = true;let nextPageData: URLSearchParams | null = params;@@ -149,7 +150,7 @@ export async function ddgSearch(response = await undici.fetch(`https://html.duckduckgo.com/html?${params.toString()}`,{- dispatcher: getSecureDispatcher(false),+ dispatcher,redirect: "follow",headers: {"User-Agent": userAgent,@@ -166,7 +167,7 @@ export async function ddgSearch(response = await undici.fetch(`https://html.duckduckgo.com/html`, {method: "POST",body: nextPageData.toString(),- dispatcher: getSecureDispatcher(false),+ dispatcher,redirect: "follow",headers: {"User-Agent": userAgent,