Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sitemap): change sitemap logging
Móricz Gergő6557365149fc
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 643b274e5..9ceaf4342 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -632,6 +632,10 @@ export class WebCrawler {}}+ if (this.sitemapsHit.size >= 20) {+ this.logger.warn("Sitemap limit hit!", { crawlId: this.jobId, url: this.baseUrl });+ }+return sitemapCount;}}diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 77db0db2a..d55efeea2 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -22,7 +22,6 @@ export async function getLinksFromSitemap(sitemapsHit: Set<string>,): Promise<number> {if (sitemapsHit.size >= 20) {- logger.warn("Sitemap limit hit, not hitting this one.");return 0;}