Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): scope fetch state

mogery656bffcc2883
diff --git a/apps/api/src/lib/validateUrl.ts b/apps/api/src/lib/validateUrl.ts
index a73c06f0d..3dcdd7119 100644
--- a/apps/api/src/lib/validateUrl.ts
+++ b/apps/api/src/lib/validateUrl.ts
@@ -200,6 +200,7 @@ export async function resolveRedirects(
const targetUrl = protocolIncluded(url) ? url : `http://${url}`;
const methods = ["HEAD", "GET"] as const;
+ const dispatcher = getSecureDispatcher(false);
for (const method of methods) {
const signal = abort
@@ -210,7 +211,7 @@ export async function resolveRedirects(
const response = await undici.fetch(targetUrl, {
method,
redirect: "follow",
- dispatcher: getSecureDispatcher(false),
+ dispatcher,
signal,
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
index 3b0ad57e5..4ef7ca8ad 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
@@ -60,11 +60,8 @@ function attachSecurityCheck(agent: undici.Dispatcher) {
});
}
-// Dispatcher WITH cookie handling (for scraping - needs cookies for auth flows)
function makeSecureDispatcher(skipTlsVerification: boolean) {
- const baseAgent = createBaseAgent(skipTlsVerification);
- const cookieJar = new CookieJar();
- const agent = baseAgent.compose(cookie({ jar: cookieJar }));
+ const agent = createBaseAgent(skipTlsVerification);
attachSecurityCheck(agent);
return agent;
}
@@ -82,8 +79,13 @@ const secureDispatcherNoCookies = makeSecureDispatcherNoCookies(false);
const secureDispatcherNoCookiesSkipTlsVerification =
makeSecureDispatcherNoCookies(true);
-export const getSecureDispatcher = (skipTlsVerification: boolean = false) =>
- skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher;
+export const getSecureDispatcher = (skipTlsVerification: boolean = false) => {
+ const dispatcher = skipTlsVerification
+ ? secureDispatcherSkipTlsVerification
+ : secureDispatcher;
+
+ return dispatcher.compose(cookie({ jar: new CookieJar() }));
+};
// Use this for webhook delivery to avoid sending empty cookie headers
export const getSecureDispatcherNoCookies = (
diff --git a/apps/api/src/search/v2/ddgsearch.ts b/apps/api/src/search/v2/ddgsearch.ts
index 9838423ac..9b2571f4f 100644
--- a/apps/api/src/search/v2/ddgsearch.ts
+++ b/apps/api/src/search/v2/ddgsearch.ts
@@ -130,6 +130,7 @@ export async function ddgSearch(
const results: WebSearchResult[] = [];
const seenUrls = new Set<string>();
+ const dispatcher = getSecureDispatcher(false);
let isFirstPage = true;
let nextPageData: URLSearchParams | null = params;
@@ -149,7 +150,7 @@ export async function ddgSearch(
response = await undici.fetch(
`https://html.duckduckgo.com/html?${params.toString()}`,
{
- dispatcher: getSecureDispatcher(false),
+ dispatcher,
redirect: "follow",
headers: {
"User-Agent": userAgent,
@@ -166,7 +167,7 @@ export async function ddgSearch(
response = await undici.fetch(`https://html.duckduckgo.com/html`, {
method: "POST",
body: nextPageData.toString(),
- dispatcher: getSecureDispatcher(false),
+ dispatcher,
redirect: "follow",
headers: {
"User-Agent": userAgent,