Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick
Nicolas45ce12c85901
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex d12ea4a75..258c1fc49 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -49,7 +49,6 @@ export function makeSecureDispatcher(const agentOpts: undici.Agent.Options = {connect: {rejectUnauthorized: false, // bypass SSL failures -- this is fine- lookup: cacheableLookup.lookup,},maxRedirections: 5000,...options,diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex ba9ec7036..7d3babd67 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -98,7 +98,7 @@ export async function robustFetch<headersTimeout: 0,bodyTimeout: 0,connect: {- lookup: cacheableLookup.lookup,+ // lookup: cacheableLookup.lookup,},}),...(body instanceof FormData