Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(sitemap): increase limit to 20
Móricz Gergő434a435a4bcc
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 7e0c6f8a2..77db0db2a 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -21,7 +21,7 @@ export async function getLinksFromSitemap(crawlId: string,sitemapsHit: Set<string>,): Promise<number> {- if (sitemapsHit.size >= 10) {+ if (sitemapsHit.size >= 20) {logger.warn("Sitemap limit hit, not hitting this one.");return 0;}