Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawl-status): improve finished checking

Gergő Móricz078c0679aa8d
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 02046364c..694b3dcaa 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -13,6 +13,7 @@ import {
getDoneJobsOrderedLength,
getThrottledJobs,
isCrawlFinished,
+ isCrawlFinishedLocked,
} from "../../lib/crawl-redis";
import { getScrapeQueue } from "../../services/queue-service";
import {
@@ -117,7 +118,7 @@ export async function crawlStatusController(
sc.cancelled
? "cancelled"
: validJobStatuses.every((x) => x[1] === "completed") &&
- await isCrawlFinished(req.params.jobId)
+ (await isCrawlFinishedLocked(req.params.jobId) || await isCrawlFinished(req.params.jobId))
? "completed"
: "scraping";