Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker/kickoff): make crawls wait for kickoff to finish (matters on big sitemapped sites)

Gergő Móriczd5929af01094
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 5b25969e5..5868c28a8 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -128,6 +128,7 @@ export async function isCrawlFinished(id: string) {
return (
(await redisConnection.scard("crawl:" + id + ":jobs_done")) ===
(await redisConnection.scard("crawl:" + id + ":jobs"))
+ && (await redisConnection.get("crawl:" + id + ":kickoff:finish")) !== null
);
}
@@ -135,6 +136,10 @@ export async function isCrawlFinishedLocked(id: string) {
return await redisConnection.exists("crawl:" + id + ":finish");
}
+export async function finishCrawlKickoff(id: string) {
+ await redisConnection.set("crawl:" + id + ":kickoff:finish", "yes", "EX", 24 * 60 * 60);
+}
+
export async function finishCrawl(id: string) {
if (await isCrawlFinished(id)) {
_logger.debug("Marking crawl as finished.", {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index c5b164f10..47f873ec3 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -23,6 +23,7 @@ import {
addCrawlJobs,
crawlToCrawler,
finishCrawl,
+ finishCrawlKickoff,
generateURLPermutations,
getCrawl,
getCrawlJobCount,
@@ -675,6 +676,9 @@ async function processKickoffJob(job: Job & { id: string }, token: string) {
logger.debug("Done queueing jobs!");
+ await finishCrawlKickoff(job.data.crawl_id);
+ await finishCrawlIfNeeded(job, sc);
+
return { success: true };
} catch (error) {
logger.error("An error occurred!", { error });