Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Disable Concurrency Limit on Self Hosted (#2475)
Abimael Martell9e98cf01763c
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex fe0c62ace..c0ae0280b 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -22,6 +22,7 @@ import { deserializeTransportableError } from "../lib/error-serde";import { abTestJob } from "./ab-test";import { NuQJob, scrapeQueue } from "./worker/nuq";import { serializeTraceContext } from "../lib/otel-tracer";+import { isSelfHosted } from "../lib/deployment";/*** Checks if a job is a crawl or batch scrape based on its options@@ -216,7 +217,10 @@ async function addScrapeJobRaw(let currentActiveConcurrency = 0;let maxConcurrency = 0;- if (directToBullMQ) {+ // Bypass concurrency limits for self-hosted deployments+ if (isSelfHosted()) {+ concurrencyLimited = "no";+ } else if (directToBullMQ) {concurrencyLimited = "no";} else {if (webScraperOptions.crawl_id) {@@ -442,36 +446,51 @@ export async function addScrapeJobs(// All jobs without a crawl ID may be in the CQ depending on the global team concurrency limitjobsPotentiallyInCQ.push(...jobsWithoutCrawlID);- const now = Date.now();- const maxConcurrency =- (- await getACUCTeam(- teamId,- false,- true,- jobs[0].data.mode === "single_urls" && jobs[0].data.from_extract- ? RateLimiterMode.Extract- : RateLimiterMode.Crawl,- )- )?.concurrency ?? 2;- await cleanOldConcurrencyLimitEntries(teamId, now);-- const currentActiveConcurrency = (- await getConcurrencyLimitActiveJobs(teamId, now)- ).length;-- const countCanBeDirectlyAdded = Math.max(- maxConcurrency - currentActiveConcurrency,- 0,- );+ // Bypass concurrency limits for self-hosted deployments+ let addToBull: typeof jobsPotentiallyInCQ;+ let addToCQ: typeof jobsPotentiallyInCQ;+ let maxConcurrency = 0;+ let countCanBeDirectlyAdded = 0;- const addToBull = jobsPotentiallyInCQ.slice(0, countCanBeDirectlyAdded);- const addToCQ = jobsPotentiallyInCQ- .slice(countCanBeDirectlyAdded)- .concat(jobsForcedToCQ);+ if (isSelfHosted()) {+ // For self-hosted, add all jobs directly to BullMQ+ addToBull = jobsPotentiallyInCQ;+ addToCQ = jobsForcedToCQ;+ } else {+ const now = Date.now();+ maxConcurrency =+ (+ await getACUCTeam(+ teamId,+ false,+ true,+ jobs[0].data.mode === "single_urls" && jobs[0].data.from_extract+ ? RateLimiterMode.Extract+ : RateLimiterMode.Crawl,+ )+ )?.concurrency ?? 2;+ await cleanOldConcurrencyLimitEntries(teamId, now);++ const currentActiveConcurrency = (+ await getConcurrencyLimitActiveJobs(teamId, now)+ ).length;++ countCanBeDirectlyAdded = Math.max(+ maxConcurrency - currentActiveConcurrency,+ 0,+ );++ addToBull = jobsPotentiallyInCQ.slice(0, countCanBeDirectlyAdded);+ addToCQ = jobsPotentiallyInCQ+ .slice(countCanBeDirectlyAdded)+ .concat(jobsForcedToCQ);+ }- // equals 2x the max concurrency- if (jobsPotentiallyInCQ.length - countCanBeDirectlyAdded > maxConcurrency) {+ // equals 2x the max concurrency (only check for non-self-hosted)+ if (+ !isSelfHosted() &&+ jobsPotentiallyInCQ.length - countCanBeDirectlyAdded > maxConcurrency+ ) {// logger.info(`Concurrency limited 2x (multiple) - Concurrency queue jobs: ${addToCQ.length} Max concurrency: ${maxConcurrency} Team ID: ${jobs[0].data.team_id}`);// Only send notification if it's not a crawl or batch scrapeif (!isCrawlOrBatchScrape(jobs[0].data)) {diff --git a/apps/api/src/services/worker/team-semaphore.ts b/apps/api/src/services/worker/team-semaphore.tsindex 3366f7927..2829c05c0 100644--- a/apps/api/src/services/worker/team-semaphore.ts+++ b/apps/api/src/services/worker/team-semaphore.ts@@ -183,7 +183,8 @@ async function withSemaphore<T>(timeoutMs: number,func: (limited: boolean) => Promise<T>,): Promise<T> {- if (isSelfHosted() && limit <= 1) {+ // Bypass concurrency limits for self-hosted deployments+ if (isSelfHosted()) {_logger.debug(`Bypassing concurrency limit for ${teamId}`, {teamId,jobId: holderId,