Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(sitemap): increase limit
Móricz Gergő1e28ba291e42
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex b6b132aa5..7e0c6f8a2 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -21,8 +21,8 @@ export async function getLinksFromSitemap(crawlId: string,sitemapsHit: Set<string>,): Promise<number> {- if (sitemapsHit.size >= 5) {- logger.warn("Sitemap limit of 5 hit, not hitting this one.");+ if (sitemapsHit.size >= 10) {+ logger.warn("Sitemap limit hit, not hitting this one.");return 0;}