Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawler): dumb sitemap limit

Móricz Gergőa922aac80594
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 442a4f5e1..f883e4c58 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -27,6 +27,7 @@ export class WebCrawler {
private allowSubdomains: boolean;
private ignoreRobotsTxt: boolean;
private logger: typeof _logger;
+ private sitemapsHit: Set<string> = new Set();
constructor({
jobId,
@@ -531,10 +532,22 @@ export class WebCrawler {
url: string,
urlsHandler: (urls: string[]) => unknown,
): Promise<number> {
+ if (this.sitemapsHit.size >= 5) {
+ this.logger.warn("Sitemap limit of 5 hit, not hitting this one.");
+ return 0;
+ }
+
const sitemapUrl = url.endsWith(".xml")
? url
: `${url}${url.endsWith("/") ? "" : "/"}sitemap.xml`;
+ if (this.sitemapsHit.has(sitemapUrl)) {
+ this.logger.warn("This sitemap has already been hit.", { sitemapUrl });
+ return 0;
+ }
+
+ this.sitemapsHit.add(sitemapUrl);
+
let sitemapCount: number = 0;
// Try to get sitemap from the provided URL first