Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "debug logging"
Gergő Móriczef33226c2fcc
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 44b2296ae..885be5005 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -86,12 +86,7 @@ export async function _addScrapeJobToBullMQ(}}- return await scrapeQueue.addJob(- jobId,- webScraperOptions,- priority,- listenable,- );+ return await scrapeQueue.addJob(jobId, webScraperOptions, priority, listenable);}async function addScrapeJobRaw(@@ -100,7 +95,6 @@ async function addScrapeJobRaw(priority: number = 0,directToBullMQ: boolean = false,listenable: boolean = false,- logger: Logger = _logger,): Promise<NuQJob<ScrapeJobData> | null> {let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;let currentActiveConcurrency = 0;@@ -108,11 +102,9 @@ async function addScrapeJobRaw(if (directToBullMQ) {concurrencyLimited = "no";- logger.debug("Adding job directly to BullMQ");} else {if (webScraperOptions.crawl_id) {const crawl = await getCrawl(webScraperOptions.crawl_id);- logger.debug("Got crawl");const concurrencyLimit = !crawl? null: crawl.crawlerOptions?.delay === undefined &&@@ -124,7 +116,6 @@ async function addScrapeJobRaw(const crawlConcurrency = (await getCrawlConcurrencyLimitActiveJobs(webScraperOptions.crawl_id)).length;- logger.debug("Got crawl concurrency limit active jobs");const freeSlots = Math.max(concurrencyLimit - crawlConcurrency, 0);if (freeSlots === 0) {concurrencyLimited = "yes-crawl";@@ -146,13 +137,10 @@ async function addScrapeJobRaw(: RateLimiterMode.Crawl,))?.concurrency ?? 2;- logger.debug("Got team ACUC concurrency limit");await cleanOldConcurrencyLimitEntries(webScraperOptions.team_id, now);- logger.debug("Cleaned old concurrency limit entries");const currentActiveConcurrency = (await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now)).length;- logger.debug("Got current active concurrency");concurrencyLimited =currentActiveConcurrency >= maxConcurrency ? "yes" : "no";}@@ -166,7 +154,6 @@ async function addScrapeJobRaw(const concurrencyQueueJobs = await getConcurrencyQueueJobsCount(webScraperOptions.team_id,);- logger.debug("Got concurrency queue jobs");if (concurrencyQueueJobs > maxConcurrency) {// logger.info("Concurrency limited 2x (single) - ", "Concurrency queue jobs: ", concurrencyQueueJobs, "Max concurrency: ", maxConcurrency, "Team ID: ", webScraperOptions.team_id);@@ -175,7 +162,6 @@ async function addScrapeJobRaw(await shouldSendConcurrencyLimitNotification(webScraperOptions.team_id,);- logger.debug("Got should send notification");if (shouldSendNotification) {sendNotificationWithCustomDays(webScraperOptions.team_id,@@ -195,23 +181,10 @@ async function addScrapeJobRaw(webScraperOptions.concurrencyLimited = true;- await _addScrapeJobToConcurrencyQueue(- webScraperOptions,- jobId,- priority,- listenable,- );- logger.debug("Added job to concurrency queue");+ await _addScrapeJobToConcurrencyQueue(webScraperOptions, jobId, priority, listenable);return null;} else {- const x = await _addScrapeJobToBullMQ(- webScraperOptions,- jobId,- priority,- listenable,- );- logger.debug("Added job to BullMQ");- return x;+ return await _addScrapeJobToBullMQ(webScraperOptions, jobId, priority, listenable);}}@@ -221,9 +194,6 @@ export async function addScrapeJob(priority: number = 0,directToBullMQ: boolean = false,listenable: boolean = false,- logger: Logger = _logger.child({- zeroDataRetention: !!webScraperOptions.zeroDataRetention,- }),): Promise<NuQJob<ScrapeJobData> | null> {// Capture trace context to propagate to workerconst traceContext = serializeTraceContext();@@ -238,7 +208,6 @@ export async function addScrapeJob(priority,directToBullMQ,listenable,- logger,);}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex ccf54f6e6..674929d76 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -373,9 +373,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {},jobId,jobPriority,- false,- false,- logger,);await addCrawlJob(job.data.crawl_id, jobId, logger);