Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker/crawl): only report successful page count in num_docs (#1179)
Gergő Móricz73e7884df4ba
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex c4ebe9398..b6d101f76 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -29,6 +29,7 @@ import {getCrawl,getCrawlJobCount,getCrawlJobs,+ getDoneJobsOrderedLength,lockURL,lockURLs,lockURLsIndividually,@@ -185,7 +186,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {);}} else {- const num_docs = await getCrawlJobCount(job.data.crawl_id);+ const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);const jobStatus = sc.cancelled ? "failed" : "completed";await logJob(