Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(WebScraper/tryGetSitemap): deduplicate sitemap links list
Gergő Móricze97ee4a4be9b
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 19b0b5b48..2e47d3526 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -210,7 +210,7 @@ export class WebCrawler {}if (sitemapLinks.length > 0) {let filteredLinks = this.filterLinks(- sitemapLinks,+ [...new Set(sitemapLinks)],this.limit,this.maxCrawledDepth,fromMap,