Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker/crawl): use SCARD to generate num_docs field
Móricz Gergő49e584f8e194
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 80720bc6e..a200b9b51 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -160,6 +160,10 @@ export async function getCrawlJobs(id: string): Promise<string[]> {return await redisConnection.smembers("crawl:" + id + ":jobs");}+export async function getCrawlJobCount(id: string): Promise<number> {+ return await redisConnection.scard("crawl:" + id + ":jobs");+}+export async function getThrottledJobs(teamId: string): Promise<string[]> {return await redisConnection.zrangebyscore("concurrency-limiter:" + teamId + ":throttled",diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex f28f3f35e..d8d154cc6 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -25,6 +25,7 @@ import {finishCrawl,generateURLPermutations,getCrawl,+ getCrawlJobCount,getCrawlJobs,lockURL,lockURLs,@@ -212,7 +213,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {);}} else {- const jobIDs = await getCrawlJobs(job.data.crawl_id);+ const num_docs = await getCrawlJobCount(job.data.crawl_id);const jobStatus = sc.cancelled ? "failed" : "completed";await logJob(@@ -220,7 +221,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {job_id: job.data.crawl_id,success: jobStatus === "completed",message: sc.cancelled ? "Cancelled" : undefined,- num_docs: jobIDs.length,+ num_docs,docs: [],time_taken: (Date.now() - sc.createdAt) / 1000,team_id: job.data.team_id,