Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: sitemap fetch only below threshold for /map
Nicolasac0d10c45171
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 75e253ba9..da12980cb 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -155,15 +155,19 @@ export async function getMapResults({await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours}- // Parallelize sitemap fetch with serper search and sitemap-index- const [_, sitemapIndexUrls, ...searchResults] = await Promise.all([- ignoreSitemap ? null : crawler.tryGetSitemap(urls => {- links.push(...urls);- }, true, false, 30000),+ // Parallelize sitemap index query with search results+ const [sitemapIndexUrls, ...searchResults] = await Promise.all([querySitemapIndex(url),...(cachedResult ? [] : pagePromises),]);+ // Only query sitemap if index has less than 100 links+ if (!ignoreSitemap && sitemapIndexUrls.length < 100) {+ await crawler.tryGetSitemap(urls => {+ links.push(...urls);+ }, true, false, 30000);+ }+if (!cachedResult) {allResults = searchResults;}