Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(robustFetch): use pre-created agent

Gergő Móriczbb5a3d48cd98
diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
index 1e4897cea..2c1007d6c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
@@ -5,7 +5,7 @@ import { TimeoutError } from "../../error";
import { specialtyScrapeCheck } from "../utils/specialtyHandler";
import {
InsecureConnectionError,
- makeSecureDispatcher,
+ secureDispatcher,
} from "../utils/safeFetch";
import { MockState, saveMock } from "../../lib/mock";
import { TextDecoder } from "util";
@@ -56,7 +56,7 @@ export async function scrapeURLWithFetch(
try {
const x = await Promise.race([
undici.fetch(meta.rewrittenUrl ?? meta.url, {
- dispatcher: await makeSecureDispatcher(meta.rewrittenUrl ?? meta.url),
+ dispatcher: secureDispatcher,
redirect: "follow",
headers: meta.options.headers,
signal: meta.internalOptions.abort ?? AbortSignal.timeout(timeout),
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
index e83cf7918..4ccd52c53 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
@@ -5,7 +5,7 @@ import { EngineError } from "../../error";
import { Writable } from "stream";
import { v4 as uuid } from "uuid";
import * as undici from "undici";
-import { makeSecureDispatcher } from "./safeFetch";
+import { secureDispatcher } from "./safeFetch";
export async function fetchFileToBuffer(
url: string,
@@ -17,7 +17,7 @@ export async function fetchFileToBuffer(
const response = await undici.fetch(url, {
...init,
redirect: "follow",
- dispatcher: await makeSecureDispatcher(url),
+ dispatcher: secureDispatcher,
});
return {
response,
@@ -40,7 +40,7 @@ export async function downloadFile(
const response = await undici.fetch(url, {
...init,
redirect: "follow",
- dispatcher: await makeSecureDispatcher(url),
+ dispatcher: secureDispatcher,
});
// This should never happen in the current state of JS/Undici (2024), but let's check anyways.
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
index d12ea4a75..e4e778999 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
@@ -42,17 +42,13 @@ function isIPv6Private(ipv6) {
return new Address6(ipv6).getScope() !== "Global";
}
-export function makeSecureDispatcher(
- url: string,
- options?: undici.Agent.Options,
-) {
+function makeSecureDispatcher() {
const agentOpts: undici.Agent.Options = {
connect: {
rejectUnauthorized: false, // bypass SSL failures -- this is fine
lookup: cacheableLookup.lookup,
},
maxRedirections: 5000,
- ...options,
};
const baseAgent = process.env.PROXY_SERVER
@@ -90,3 +86,5 @@ export function makeSecureDispatcher(
return agent;
}
+
+export const secureDispatcher = makeSecureDispatcher();
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index 487bf63f6..4d23c77a8 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -27,6 +27,22 @@ export type RobustFetchParams<Schema extends z.Schema<any>> = {
useCacheableLookup?: boolean;
};
+const robustAgent = new Agent({
+ headersTimeout: 0,
+ bodyTimeout: 0,
+ connect: {
+ lookup: cacheableLookup.lookup,
+ },
+});
+
+const robustAgentNoLookup = new Agent({
+ headersTimeout: 0,
+ bodyTimeout: 0,
+ connect: {
+ lookup: dns.lookup,
+ },
+});
+
export async function robustFetch<
Schema extends z.Schema<any>,
Output = z.infer<Schema>,
@@ -97,13 +113,7 @@ export async function robustFetch<
...(headers !== undefined ? headers : {}),
},
signal: abort,
- dispatcher: new Agent({
- headersTimeout: 0,
- bodyTimeout: 0,
- connect: {
- lookup: useCacheableLookup ? cacheableLookup.lookup : dns.lookup,
- },
- }),
+ dispatcher: useCacheableLookup ? robustAgent : robustAgentNoLookup,
...(body instanceof FormData
? {
body,