Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker/crawl): use SCARD to generate num_docs field

Móricz Gergő49e584f8e194
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 80720bc6e..a200b9b51 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -160,6 +160,10 @@ export async function getCrawlJobs(id: string): Promise<string[]> {
return await redisConnection.smembers("crawl:" + id + ":jobs");
}
+export async function getCrawlJobCount(id: string): Promise<number> {
+ return await redisConnection.scard("crawl:" + id + ":jobs");
+}
+
export async function getThrottledJobs(teamId: string): Promise<string[]> {
return await redisConnection.zrangebyscore(
"concurrency-limiter:" + teamId + ":throttled",
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index f28f3f35e..d8d154cc6 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -25,6 +25,7 @@ import {
finishCrawl,
generateURLPermutations,
getCrawl,
+ getCrawlJobCount,
getCrawlJobs,
lockURL,
lockURLs,
@@ -212,7 +213,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
);
}
} else {
- const jobIDs = await getCrawlJobs(job.data.crawl_id);
+ const num_docs = await getCrawlJobCount(job.data.crawl_id);
const jobStatus = sc.cancelled ? "failed" : "completed";
await logJob(
@@ -220,7 +221,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
job_id: job.data.crawl_id,
success: jobStatus === "completed",
message: sc.cancelled ? "Cancelled" : undefined,
- num_docs: jobIDs.length,
+ num_docs,
docs: [],
time_taken: (Date.now() - sc.createdAt) / 1000,
team_id: job.data.team_id,