Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker/crawl): only report successful page count in num_docs (#1179)

Gergő Móricz73e7884df4ba
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index c4ebe9398..b6d101f76 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -29,6 +29,7 @@ import {
getCrawl,
getCrawlJobCount,
getCrawlJobs,
+ getDoneJobsOrderedLength,
lockURL,
lockURLs,
lockURLsIndividually,
@@ -185,7 +186,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
);
}
} else {
- const num_docs = await getCrawlJobCount(job.data.crawl_id);
+ const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);
const jobStatus = sc.cancelled ? "failed" : "completed";
await logJob(