Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore(api): log concurrency queue inserts
mogery10588364fb59
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 3e0cf1fcb..377393edd 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -234,7 +234,7 @@ async function addScrapeJobRaw(listenable: boolean = false,): Promise<NuQJob<ScrapeJobData> | null> {let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;- let currentActiveConcurrency = 0;+ let currentActiveConcurrency: number | null = null;let maxConcurrency = 0;// Bypass concurrency limits for self-hosted deployments@@ -294,6 +294,21 @@ async function addScrapeJobRaw(throw new QueueFullError(concurrencyQueueJobs, queueLimit);}+ if (currentActiveConcurrency === null) {+ const now = Date.now();+ await cleanOldConcurrencyLimitEntries(webScraperOptions.team_id, now);+ currentActiveConcurrency = (+ await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now)+ ).length;+ }++ _logger.info("Adding scrape job to concurrency queue", {+ teamId: webScraperOptions.team_id,+ maxConcurrency,+ currentConcurrency: currentActiveConcurrency,+ jobId,+ });+if (concurrencyLimited === "yes") {// Detect if they hit their concurrent limit// If above by 2x, send them an email@@ -474,6 +489,7 @@ export async function addScrapeJobs(let addToBull: typeof jobsPotentiallyInCQ;let addToCQ: typeof jobsPotentiallyInCQ;let maxConcurrency = 0;+ let currentActiveConcurrency: number | null = null;let countCanBeDirectlyAdded = 0;if (isSelfHosted()) {@@ -487,7 +503,7 @@ export async function addScrapeJobs(?.concurrency ?? 2;await cleanOldConcurrencyLimitEntries(teamId, now);- const currentActiveConcurrency = (+ currentActiveConcurrency = (await getConcurrencyLimitActiveJobs(teamId, now)).length;@@ -510,6 +526,23 @@ export async function addScrapeJobs(}}+ if (addToCQ.length > 0) {+ if (currentActiveConcurrency === null) {+ const now = Date.now();+ await cleanOldConcurrencyLimitEntries(teamId, now);+ currentActiveConcurrency = (+ await getConcurrencyLimitActiveJobs(teamId, now)+ ).length;+ }++ _logger.info("Adding scrape jobs to concurrency queue", {+ teamId,+ maxConcurrency,+ currentConcurrency: currentActiveConcurrency,+ jobsCount: addToCQ.length,+ });+ }+// equals 2x the max concurrency (only check for non-self-hosted)if (!isSelfHosted() &&