Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker/kickoff): make crawls wait for kickoff to finish (matters on big sitemapped sites)
Gergő Móriczd5929af01094
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 5b25969e5..5868c28a8 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -128,6 +128,7 @@ export async function isCrawlFinished(id: string) {return ((await redisConnection.scard("crawl:" + id + ":jobs_done")) ===(await redisConnection.scard("crawl:" + id + ":jobs"))+ && (await redisConnection.get("crawl:" + id + ":kickoff:finish")) !== null);}@@ -135,6 +136,10 @@ export async function isCrawlFinishedLocked(id: string) {return await redisConnection.exists("crawl:" + id + ":finish");}+export async function finishCrawlKickoff(id: string) {+ await redisConnection.set("crawl:" + id + ":kickoff:finish", "yes", "EX", 24 * 60 * 60);+}+export async function finishCrawl(id: string) {if (await isCrawlFinished(id)) {_logger.debug("Marking crawl as finished.", {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex c5b164f10..47f873ec3 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -23,6 +23,7 @@ import {addCrawlJobs,crawlToCrawler,finishCrawl,+ finishCrawlKickoff,generateURLPermutations,getCrawl,getCrawlJobCount,@@ -675,6 +676,9 @@ async function processKickoffJob(job: Job & { id: string }, token: string) {logger.debug("Done queueing jobs!");+ await finishCrawlKickoff(job.data.crawl_id);+ await finishCrawlIfNeeded(job, sc);+return { success: true };} catch (error) {logger.error("An error occurred!", { error });