Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(WebScraper/sitemap): await urlsHandler to fix race condition
Gergő Móricz71a8f7452cb5
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 5695d2844..9c49cf9b8 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -506,7 +506,7 @@ export class WebCrawler {// Get all links from the main domain's sitemapsitemapCount += await getLinksFromSitemap({ sitemapUrl: mainDomainSitemapUrl, urlsHandler(urls) {- urlsHandler(urls.filter(link => {+ return urlsHandler(urls.filter(link => {try {const linkUrl = new URL(link);return linkUrl.hostname.endsWith(hostname);diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 8028d225c..128b3f037 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -103,7 +103,11 @@ export async function getLinksFromSitemap().map((url) => url.loc[0]);count += validUrls.length;- urlsHandler(validUrls);++ const h = urlsHandler(validUrls);+ if (h instanceof Promise) {+ await h;+ }}return count;