Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(queue-worker): fix crawl pre-finishing logic (#1628)
Gergő Móricz0f394a10c620
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 0984a6286..bcaa76a81 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -162,16 +162,17 @@ export async function finishCrawlPre(id: string) {const set = await redisEvictConnection.setnx("crawl:" + id + ":finished_pre", "yes");await redisEvictConnection.expire("crawl:" + id + ":finished_pre", 24 * 60 * 60);return set === 1;- } else {- // _logger.debug("Crawl can not be pre-finished yet, not marking as finished.", {- // module: "crawl-redis",- // method: "finishCrawlPre",- // crawlId: id,- // jobs_done: await redisEvictConnection.scard("crawl:" + id + ":jobs_done"),- // jobs: await redisEvictConnection.scard("crawl:" + id + ":jobs"),- // kickoff_finished:- // (await redisEvictConnection.get("crawl:" + id + ":kickoff:finish")) !== null,- // });+ }+}++export async function unPreFinishCrawl(id: string) {+ if (await isCrawlFinished(id)) {+ _logger.debug("Un-pre-finishing crawl.", {+ module: "crawl-redis",+ method: "unPreFinishCrawl",+ crawlId: id,+ });+ await redisEvictConnection.del("crawl:" + id + ":finished_pre");}}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 2a1bdf665..2404e9171 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -26,6 +26,7 @@ import {crawlToCrawler,finishCrawl,finishCrawlPre,+ unPreFinishCrawl,finishCrawlKickoff,generateURLPermutations,getCrawl,@@ -247,6 +248,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {lockedJobs: lockedJobs.length,});+ await unPreFinishCrawl(job.data.crawl_id);return;} else {logger.info("No jobs added (all discovered URLs were locked), finishing crawl");