Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

chore(api): log concurrency queue inserts

mogery10588364fb59
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 3e0cf1fcb..377393edd 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -234,7 +234,7 @@ async function addScrapeJobRaw(
listenable: boolean = false,
): Promise<NuQJob<ScrapeJobData> | null> {
let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;
- let currentActiveConcurrency = 0;
+ let currentActiveConcurrency: number | null = null;
let maxConcurrency = 0;
// Bypass concurrency limits for self-hosted deployments
@@ -294,6 +294,21 @@ async function addScrapeJobRaw(
throw new QueueFullError(concurrencyQueueJobs, queueLimit);
}
+ if (currentActiveConcurrency === null) {
+ const now = Date.now();
+ await cleanOldConcurrencyLimitEntries(webScraperOptions.team_id, now);
+ currentActiveConcurrency = (
+ await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now)
+ ).length;
+ }
+
+ _logger.info("Adding scrape job to concurrency queue", {
+ teamId: webScraperOptions.team_id,
+ maxConcurrency,
+ currentConcurrency: currentActiveConcurrency,
+ jobId,
+ });
+
if (concurrencyLimited === "yes") {
// Detect if they hit their concurrent limit
// If above by 2x, send them an email
@@ -474,6 +489,7 @@ export async function addScrapeJobs(
let addToBull: typeof jobsPotentiallyInCQ;
let addToCQ: typeof jobsPotentiallyInCQ;
let maxConcurrency = 0;
+ let currentActiveConcurrency: number | null = null;
let countCanBeDirectlyAdded = 0;
if (isSelfHosted()) {
@@ -487,7 +503,7 @@ export async function addScrapeJobs(
?.concurrency ?? 2;
await cleanOldConcurrencyLimitEntries(teamId, now);
- const currentActiveConcurrency = (
+ currentActiveConcurrency = (
await getConcurrencyLimitActiveJobs(teamId, now)
).length;
@@ -510,6 +526,23 @@ export async function addScrapeJobs(
}
}
+ if (addToCQ.length > 0) {
+ if (currentActiveConcurrency === null) {
+ const now = Date.now();
+ await cleanOldConcurrencyLimitEntries(teamId, now);
+ currentActiveConcurrency = (
+ await getConcurrencyLimitActiveJobs(teamId, now)
+ ).length;
+ }
+
+ _logger.info("Adding scrape jobs to concurrency queue", {
+ teamId,
+ maxConcurrency,
+ currentConcurrency: currentActiveConcurrency,
+ jobsCount: addToCQ.length,
+ });
+ }
+
// equals 2x the max concurrency (only check for non-self-hosted)
if (
!isSelfHosted() &&