Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(unvisitedUrls): filter with crawler
Gergő Móricz570809aa59ea
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex a7da10f77..36f8ed2da 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -118,7 +118,19 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {const lastUrlsSet = new Set(lastUrls);- const univistedUrls = Array.from(lastUrlsSet).filter(x => !visitedUrls.has(x));+ const crawler = crawlToCrawler(+ job.data.crawl_id,+ sc,+ sc.originUrl!,+ job.data.crawlerOptions,+ );++ const univistedUrls = crawler.filterLinks(+ Array.from(lastUrlsSet).filter(x => !visitedUrls.has(x)),+ Infinity,+ sc.crawlerOptions.maxDepth ?? 10,+ );+const addableJobCount = sc.crawlerOptions.limit === undefined ? Infinity : (sc.crawlerOptions.limit - await getDoneJobsOrderedLength(job.data.crawl_id));console.log(sc.originUrl!, univistedUrls, visitedUrls, lastUrls, addableJobCount);