Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker): finish crawl if all addable URLs were already locked (#1582)

Gergő Móricza3aee9be13bb
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 18203f817..3b4afa290 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -239,11 +239,15 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
);
await addScrapeJobs(lockedJobs);
- logger.info("Added jobs, not going for the full finish", {
- lockedJobs: lockedJobs.length,
- });
+ if (lockedJobs.length > 0) {
+ logger.info("Added jobs, not going for the full finish", {
+ lockedJobs: lockedJobs.length,
+ });
- return;
+ return;
+ } else {
+ logger.info("No jobs added (all discovered URLs were locked), finishing crawl");
+ }
}
}