Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: concurrency limit cap (ENG-4567) (#2990)
tomsideguideb5eda3cf402c
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex bc4042b07..e2ac0b823 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -26,6 +26,7 @@ import {ResponseWithSentry,} from "./controllers/v1/types";import { ZodError } from "zod";+import { QueueFullError } from "./lib/concurrency-limit";import { v7 as uuidv7 } from "uuid";import { attachWsProxy } from "./services/agentLivecastWS";import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";@@ -173,7 +174,12 @@ app.use(res: Response<ErrorResponse>,next: NextFunction,) => {- if (err instanceof ZodError) {+ if (err instanceof QueueFullError) {+ res.status(429).json({+ success: false,+ error: err.message,+ });+ } else if (err instanceof ZodError) {// In zod v4, ZodError uses 'issues' instead of 'errors'const issues = err.issues;diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 267c4738f..3eb64510a 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -6,6 +6,21 @@ import { logger } from "./logger";import { abTestJob } from "../services/ab-test";import { scrapeQueue, type NuQJob } from "../services/worker/nuq";+export class QueueFullError extends Error {+ statusCode = 429;+ constructor(queueSize: number, queueLimit: number) {+ super(+ `Queue limit reached: your team has ${queueSize} jobs queued (limit: ${queueLimit}). Please wait for existing jobs to complete before adding more, or upgrade your plan for a higher limit. For more info, see https://docs.firecrawl.dev/rate-limits#concurrent-browser-limits`,+ );+ this.name = "QueueFullError";+ }+}++// min 50k, max 2M, 2000 per concurrent browser+export function getTeamQueueLimit(concurrencyLimit: number): number {+ return Math.min(Math.max(concurrencyLimit * 2000, 50_000), 2_000_000);+}+const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;const constructQueueKey = (team_id: string) =>"concurrency-limit-queue:" + team_id;diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 6ed5dbf03..cfb20152d 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -5,10 +5,12 @@ import {getConcurrencyLimitActiveJobs,getConcurrencyQueueJobsCount,getCrawlConcurrencyLimitActiveJobs,+ getTeamQueueLimit,pushConcurrencyLimitActiveJob,pushConcurrencyLimitedJob,pushConcurrencyLimitedJobs,pushCrawlConcurrencyLimitActiveJob,+ QueueFullError,} from "../lib/concurrency-limit";import { logger as _logger } from "../lib/logger";import { sendNotificationWithCustomDays } from "./notification/email_notification";@@ -259,19 +261,20 @@ async function addScrapeJobRaw(}}+ maxConcurrency =+ (+ await getACUCTeam(+ webScraperOptions.team_id,+ false,+ true,+ RateLimiterMode.Crawl,+ )+ )?.concurrency ?? 2;+if (concurrencyLimited === null) {const now = Date.now();- const maxConcurrency =- (- await getACUCTeam(- webScraperOptions.team_id,- false,- true,- RateLimiterMode.Crawl,- )- )?.concurrency ?? 2;await cleanOldConcurrencyLimitEntries(webScraperOptions.team_id, now);- const currentActiveConcurrency = (+ currentActiveConcurrency = (await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now)).length;concurrencyLimited =@@ -280,13 +283,19 @@ async function addScrapeJobRaw(}if (concurrencyLimited === "yes" || concurrencyLimited === "yes-crawl") {+ const concurrencyQueueJobs = await getConcurrencyQueueJobsCount(+ webScraperOptions.team_id,+ );++ const queueLimit = getTeamQueueLimit(maxConcurrency);+ if (concurrencyQueueJobs >= queueLimit) {+ throw new QueueFullError(concurrencyQueueJobs, queueLimit);+ }+if (concurrencyLimited === "yes") {// Detect if they hit their concurrent limit// If above by 2x, send them an email// No need to 2x as if there are more than the max concurrency in the concurrency queue, it is already 2x- const concurrencyQueueJobs = await getConcurrencyQueueJobsCount(- webScraperOptions.team_id,- );if (concurrencyQueueJobs > maxConcurrency) {// logger.info("Concurrency limited 2x (single) - ", "Concurrency queue jobs: ", concurrencyQueueJobs, "Max concurrency: ", maxConcurrency, "Team ID: ", webScraperOptions.team_id);@@ -489,6 +498,14 @@ export async function addScrapeJobs(addToCQ = jobsPotentiallyInCQ.slice(countCanBeDirectlyAdded).concat(jobsForcedToCQ);++ if (addToCQ.length > 0) {+ const currentQueueSize = await getConcurrencyQueueJobsCount(teamId);+ const queueLimit = getTeamQueueLimit(maxConcurrency);+ if (currentQueueSize + addToCQ.length > queueLimit) {+ throw new QueueFullError(currentQueueSize, queueLimit);+ }+ }}// equals 2x the max concurrency (only check for non-self-hosted)diff --git a/apps/api/src/services/supabase.ts b/apps/api/src/services/supabase.tsindex d6a21229f..7e99131d9 100644--- a/apps/api/src/services/supabase.ts+++ b/apps/api/src/services/supabase.ts@@ -5,7 +5,7 @@ import { logger } from "../lib/logger";configDotenv();/** PostgREST code when .single() returns 0 or >1 rows. Use to distinguish "no row" from real DB errors. */-export const POSTGREST_NO_ROWS_CODE = "PGRST116";+const POSTGREST_NO_ROWS_CODE = "PGRST116";export function isPostgrestNoRowsError(error: { code?: string } | null | undefined,