Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(queue-worker): add more logs around crawl finishing logic
Gergő Móriczfdeb01847d73
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 1c329c96e..c6bc71bb1 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -109,7 +109,16 @@ const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;const runningJobs: Set<string> = new Set();async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {+ const logger = _logger.child({+ module: "queue-worker",+ method: "finishCrawlIfNeeded",+ jobId: job.id,+ scrapeId: job.id,+ crawlId: job.data.crawl_id,+ });+if (await finishCrawlPre(job.data.crawl_id)) {+ logger.info("Crawl is pre-finished, checking if we need to add more jobs");if (job.data.crawlerOptions &&!(await redisConnection.exists(@@ -131,6 +140,10 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {),);+ logger.info("Visited URLs", {+ visitedUrls: visitedUrls.size,+ });+const lastUrls: string[] = ((await supabase_service.rpc("diff_get_last_crawl_urls", {@@ -142,6 +155,10 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {const lastUrlsSet = new Set(lastUrls);+ logger.info("Last URLs", {+ lastUrls: lastUrlsSet.size,+ });+const crawler = crawlToCrawler(job.data.crawl_id,sc,@@ -162,15 +179,12 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {: sc.crawlerOptions.limit -(await getDoneJobsOrderedLength(job.data.crawl_id));- console.log(- sc.originUrl!,- univistedUrls,- visitedUrls,- lastUrls,- addableJobCount,- );-if (univistedUrls.length !== 0 && addableJobCount > 0) {+ logger.info("Adding jobs", {+ univistedUrls: univistedUrls.length,+ addableJobCount,+ });+const jobs = univistedUrls.slice(0, addableJobCount).map((url) => {const uuid = uuidv4();return {@@ -212,10 +226,15 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {);await addScrapeJobs(lockedJobs);+ logger.info("Added jobs, not going for the full finish", {+ lockedJobs: lockedJobs.length,+ });+return;}}+ logger.info("Finishing crawl");await finishCrawl(job.data.crawl_id);(async () => {@@ -267,6 +286,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {).filter((x) => x !== null);+ logger.info("Logging crawl NOW!");await logJob({job_id: job.data.crawl_id,success: jobStatus === "completed",@@ -281,6 +301,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {crawlerOptions: sc.crawlerOptions,origin: job.data.origin,});+ logger.info("Logged crawl!");const data = {success: jobStatus !== "failed",