Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker): finish crawl if all addable URLs were already locked (#1582)
Gergő Móricza3aee9be13bb
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 18203f817..3b4afa290 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -239,11 +239,15 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {);await addScrapeJobs(lockedJobs);- logger.info("Added jobs, not going for the full finish", {- lockedJobs: lockedJobs.length,- });+ if (lockedJobs.length > 0) {+ logger.info("Added jobs, not going for the full finish", {+ lockedJobs: lockedJobs.length,+ });- return;+ return;+ } else {+ logger.info("No jobs added (all discovered URLs were locked), finishing crawl");+ }}}