Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawler): dumb sitemap limit
Móricz Gergőa922aac80594
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 442a4f5e1..f883e4c58 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -27,6 +27,7 @@ export class WebCrawler {private allowSubdomains: boolean;private ignoreRobotsTxt: boolean;private logger: typeof _logger;+ private sitemapsHit: Set<string> = new Set();constructor({jobId,@@ -531,10 +532,22 @@ export class WebCrawler {url: string,urlsHandler: (urls: string[]) => unknown,): Promise<number> {+ if (this.sitemapsHit.size >= 5) {+ this.logger.warn("Sitemap limit of 5 hit, not hitting this one.");+ return 0;+ }+const sitemapUrl = url.endsWith(".xml")? url: `${url}${url.endsWith("/") ? "" : "/"}sitemap.xml`;+ if (this.sitemapsHit.has(sitemapUrl)) {+ this.logger.warn("This sitemap has already been hit.", { sitemapUrl });+ return 0;+ }++ this.sitemapsHit.add(sitemapUrl);+let sitemapCount: number = 0;// Try to get sitemap from the provided URL first