Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(sitemap): increase limit to 20

Móricz Gergő434a435a4bcc
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 7e0c6f8a2..77db0db2a 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -21,7 +21,7 @@ export async function getLinksFromSitemap(
crawlId: string,
sitemapsHit: Set<string>,
): Promise<number> {
- if (sitemapsHit.size >= 10) {
+ if (sitemapsHit.size >= 20) {
logger.warn("Sitemap limit hit, not hitting this one.");
return 0;
}