Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(robustFetch): use pre-created agent
Gergő Móriczbb5a3d48cd98
diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.tsindex 1e4897cea..2c1007d6c 100644--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts@@ -5,7 +5,7 @@ import { TimeoutError } from "../../error";import { specialtyScrapeCheck } from "../utils/specialtyHandler";import {InsecureConnectionError,- makeSecureDispatcher,+ secureDispatcher,} from "../utils/safeFetch";import { MockState, saveMock } from "../../lib/mock";import { TextDecoder } from "util";@@ -56,7 +56,7 @@ export async function scrapeURLWithFetch(try {const x = await Promise.race([undici.fetch(meta.rewrittenUrl ?? meta.url, {- dispatcher: await makeSecureDispatcher(meta.rewrittenUrl ?? meta.url),+ dispatcher: secureDispatcher,redirect: "follow",headers: meta.options.headers,signal: meta.internalOptions.abort ?? AbortSignal.timeout(timeout),diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.tsindex e83cf7918..4ccd52c53 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts@@ -5,7 +5,7 @@ import { EngineError } from "../../error";import { Writable } from "stream";import { v4 as uuid } from "uuid";import * as undici from "undici";-import { makeSecureDispatcher } from "./safeFetch";+import { secureDispatcher } from "./safeFetch";export async function fetchFileToBuffer(url: string,@@ -17,7 +17,7 @@ export async function fetchFileToBuffer(const response = await undici.fetch(url, {...init,redirect: "follow",- dispatcher: await makeSecureDispatcher(url),+ dispatcher: secureDispatcher,});return {response,@@ -40,7 +40,7 @@ export async function downloadFile(const response = await undici.fetch(url, {...init,redirect: "follow",- dispatcher: await makeSecureDispatcher(url),+ dispatcher: secureDispatcher,});// This should never happen in the current state of JS/Undici (2024), but let's check anyways.diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex d12ea4a75..e4e778999 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -42,17 +42,13 @@ function isIPv6Private(ipv6) {return new Address6(ipv6).getScope() !== "Global";}-export function makeSecureDispatcher(- url: string,- options?: undici.Agent.Options,-) {+function makeSecureDispatcher() {const agentOpts: undici.Agent.Options = {connect: {rejectUnauthorized: false, // bypass SSL failures -- this is finelookup: cacheableLookup.lookup,},maxRedirections: 5000,- ...options,};const baseAgent = process.env.PROXY_SERVER@@ -90,3 +86,5 @@ export function makeSecureDispatcher(return agent;}++export const secureDispatcher = makeSecureDispatcher();\ No newline at end of filediff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex 487bf63f6..4d23c77a8 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -27,6 +27,22 @@ export type RobustFetchParams<Schema extends z.Schema<any>> = {useCacheableLookup?: boolean;};+const robustAgent = new Agent({+ headersTimeout: 0,+ bodyTimeout: 0,+ connect: {+ lookup: cacheableLookup.lookup,+ },+});++const robustAgentNoLookup = new Agent({+ headersTimeout: 0,+ bodyTimeout: 0,+ connect: {+ lookup: dns.lookup,+ },+});+export async function robustFetch<Schema extends z.Schema<any>,Output = z.infer<Schema>,@@ -97,13 +113,7 @@ export async function robustFetch<...(headers !== undefined ? headers : {}),},signal: abort,- dispatcher: new Agent({- headersTimeout: 0,- bodyTimeout: 0,- connect: {- lookup: useCacheableLookup ? cacheableLookup.lookup : dns.lookup,- },- }),+ dispatcher: useCacheableLookup ? robustAgent : robustAgentNoLookup,...(body instanceof FormData? {body,