Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(queue-worker): fix crawl pre-finishing logic (#1628)

Gergő Móricz0f394a10c620
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 0984a6286..bcaa76a81 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -162,16 +162,17 @@ export async function finishCrawlPre(id: string) {
const set = await redisEvictConnection.setnx("crawl:" + id + ":finished_pre", "yes");
await redisEvictConnection.expire("crawl:" + id + ":finished_pre", 24 * 60 * 60);
return set === 1;
- } else {
- // _logger.debug("Crawl can not be pre-finished yet, not marking as finished.", {
- // module: "crawl-redis",
- // method: "finishCrawlPre",
- // crawlId: id,
- // jobs_done: await redisEvictConnection.scard("crawl:" + id + ":jobs_done"),
- // jobs: await redisEvictConnection.scard("crawl:" + id + ":jobs"),
- // kickoff_finished:
- // (await redisEvictConnection.get("crawl:" + id + ":kickoff:finish")) !== null,
- // });
+ }
+}
+
+export async function unPreFinishCrawl(id: string) {
+ if (await isCrawlFinished(id)) {
+ _logger.debug("Un-pre-finishing crawl.", {
+ module: "crawl-redis",
+ method: "unPreFinishCrawl",
+ crawlId: id,
+ });
+ await redisEvictConnection.del("crawl:" + id + ":finished_pre");
}
}
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 2a1bdf665..2404e9171 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -26,6 +26,7 @@ import {
crawlToCrawler,
finishCrawl,
finishCrawlPre,
+ unPreFinishCrawl,
finishCrawlKickoff,
generateURLPermutations,
getCrawl,
@@ -247,6 +248,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
lockedJobs: lockedJobs.length,
});
+ await unPreFinishCrawl(job.data.crawl_id);
return;
} else {
logger.info("No jobs added (all discovered URLs were locked), finishing crawl");