Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "debug logging"

Gergő Móriczef33226c2fcc
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 44b2296ae..885be5005 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -86,12 +86,7 @@ export async function _addScrapeJobToBullMQ(
}
}
- return await scrapeQueue.addJob(
- jobId,
- webScraperOptions,
- priority,
- listenable,
- );
+ return await scrapeQueue.addJob(jobId, webScraperOptions, priority, listenable);
}
async function addScrapeJobRaw(
@@ -100,7 +95,6 @@ async function addScrapeJobRaw(
priority: number = 0,
directToBullMQ: boolean = false,
listenable: boolean = false,
- logger: Logger = _logger,
): Promise<NuQJob<ScrapeJobData> | null> {
let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;
let currentActiveConcurrency = 0;
@@ -108,11 +102,9 @@ async function addScrapeJobRaw(
if (directToBullMQ) {
concurrencyLimited = "no";
- logger.debug("Adding job directly to BullMQ");
} else {
if (webScraperOptions.crawl_id) {
const crawl = await getCrawl(webScraperOptions.crawl_id);
- logger.debug("Got crawl");
const concurrencyLimit = !crawl
? null
: crawl.crawlerOptions?.delay === undefined &&
@@ -124,7 +116,6 @@ async function addScrapeJobRaw(
const crawlConcurrency = (
await getCrawlConcurrencyLimitActiveJobs(webScraperOptions.crawl_id)
).length;
- logger.debug("Got crawl concurrency limit active jobs");
const freeSlots = Math.max(concurrencyLimit - crawlConcurrency, 0);
if (freeSlots === 0) {
concurrencyLimited = "yes-crawl";
@@ -146,13 +137,10 @@ async function addScrapeJobRaw(
: RateLimiterMode.Crawl,
)
)?.concurrency ?? 2;
- logger.debug("Got team ACUC concurrency limit");
await cleanOldConcurrencyLimitEntries(webScraperOptions.team_id, now);
- logger.debug("Cleaned old concurrency limit entries");
const currentActiveConcurrency = (
await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now)
).length;
- logger.debug("Got current active concurrency");
concurrencyLimited =
currentActiveConcurrency >= maxConcurrency ? "yes" : "no";
}
@@ -166,7 +154,6 @@ async function addScrapeJobRaw(
const concurrencyQueueJobs = await getConcurrencyQueueJobsCount(
webScraperOptions.team_id,
);
- logger.debug("Got concurrency queue jobs");
if (concurrencyQueueJobs > maxConcurrency) {
// logger.info("Concurrency limited 2x (single) - ", "Concurrency queue jobs: ", concurrencyQueueJobs, "Max concurrency: ", maxConcurrency, "Team ID: ", webScraperOptions.team_id);
@@ -175,7 +162,6 @@ async function addScrapeJobRaw(
await shouldSendConcurrencyLimitNotification(
webScraperOptions.team_id,
);
- logger.debug("Got should send notification");
if (shouldSendNotification) {
sendNotificationWithCustomDays(
webScraperOptions.team_id,
@@ -195,23 +181,10 @@ async function addScrapeJobRaw(
webScraperOptions.concurrencyLimited = true;
- await _addScrapeJobToConcurrencyQueue(
- webScraperOptions,
- jobId,
- priority,
- listenable,
- );
- logger.debug("Added job to concurrency queue");
+ await _addScrapeJobToConcurrencyQueue(webScraperOptions, jobId, priority, listenable);
return null;
} else {
- const x = await _addScrapeJobToBullMQ(
- webScraperOptions,
- jobId,
- priority,
- listenable,
- );
- logger.debug("Added job to BullMQ");
- return x;
+ return await _addScrapeJobToBullMQ(webScraperOptions, jobId, priority, listenable);
}
}
@@ -221,9 +194,6 @@ export async function addScrapeJob(
priority: number = 0,
directToBullMQ: boolean = false,
listenable: boolean = false,
- logger: Logger = _logger.child({
- zeroDataRetention: !!webScraperOptions.zeroDataRetention,
- }),
): Promise<NuQJob<ScrapeJobData> | null> {
// Capture trace context to propagate to worker
const traceContext = serializeTraceContext();
@@ -238,7 +208,6 @@ export async function addScrapeJob(
priority,
directToBullMQ,
listenable,
- logger,
);
}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index ccf54f6e6..674929d76 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -373,9 +373,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
},
jobId,
jobPriority,
- false,
- false,
- logger,
);
await addCrawlJob(job.data.crawl_id, jobId, logger);