Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/robots-txt): set timeout on fetch
Gergő Móricz753126b01474
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex 40712f183..656299e39 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -56,6 +56,7 @@ export async function fetchRobotsTxt(robotsTxtUrl,scrapeOptions.parse({formats: ["rawHtml"],+ timeout: 5000,...(location ? { location } : {}),}),{