Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(sitemap): increase limit

Móricz Gergő1e28ba291e42
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index b6b132aa5..7e0c6f8a2 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -21,8 +21,8 @@ export async function getLinksFromSitemap(
crawlId: string,
sitemapsHit: Set<string>,
): Promise<number> {
- if (sitemapsHit.size >= 5) {
- logger.warn("Sitemap limit of 5 hit, not hitting this one.");
+ if (sitemapsHit.size >= 10) {
+ logger.warn("Sitemap limit hit, not hitting this one.");
return 0;
}