Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawler): bad urls from sitemap
Gergő Móricz5992c5715818
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 1cca7a185..a66451524 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -228,7 +228,7 @@ export class WebCrawler {);leftOfLimit -= filteredLinks.length;let uniqueURLs: string[] = [];- for (const url of urls) {+ for (const url of filteredLinks) {if (await redisConnection.sadd("sitemap:" + this.jobId + ":links",