Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(sitemap): change sitemap logging

Móricz Gergő6557365149fc
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 643b274e5..9ceaf4342 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -632,6 +632,10 @@ export class WebCrawler {
}
}
+ if (this.sitemapsHit.size >= 20) {
+ this.logger.warn("Sitemap limit hit!", { crawlId: this.jobId, url: this.baseUrl });
+ }
+
return sitemapCount;
}
}
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 77db0db2a..d55efeea2 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -22,7 +22,6 @@ export async function getLinksFromSitemap(
sitemapsHit: Set<string>,
): Promise<number> {
if (sitemapsHit.size >= 20) {
- logger.warn("Sitemap limit hit, not hitting this one.");
return 0;
}