Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: concurrency limit cap (ENG-4567) (#2990)

tomsideguideb5eda3cf402c
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index bc4042b07..e2ac0b823 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -26,6 +26,7 @@ import {
ResponseWithSentry,
} from "./controllers/v1/types";
import { ZodError } from "zod";
+import { QueueFullError } from "./lib/concurrency-limit";
import { v7 as uuidv7 } from "uuid";
import { attachWsProxy } from "./services/agentLivecastWS";
import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";
@@ -173,7 +174,12 @@ app.use(
res: Response<ErrorResponse>,
next: NextFunction,
) => {
- if (err instanceof ZodError) {
+ if (err instanceof QueueFullError) {
+ res.status(429).json({
+ success: false,
+ error: err.message,
+ });
+ } else if (err instanceof ZodError) {
// In zod v4, ZodError uses 'issues' instead of 'errors'
const issues = err.issues;
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 267c4738f..3eb64510a 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -6,6 +6,21 @@ import { logger } from "./logger";
import { abTestJob } from "../services/ab-test";
import { scrapeQueue, type NuQJob } from "../services/worker/nuq";
+export class QueueFullError extends Error {
+ statusCode = 429;
+ constructor(queueSize: number, queueLimit: number) {
+ super(
+ `Queue limit reached: your team has ${queueSize} jobs queued (limit: ${queueLimit}). Please wait for existing jobs to complete before adding more, or upgrade your plan for a higher limit. For more info, see https://docs.firecrawl.dev/rate-limits#concurrent-browser-limits`,
+ );
+ this.name = "QueueFullError";
+ }
+}
+
+// min 50k, max 2M, 2000 per concurrent browser
+export function getTeamQueueLimit(concurrencyLimit: number): number {
+ return Math.min(Math.max(concurrencyLimit * 2000, 50_000), 2_000_000);
+}
+
const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;
const constructQueueKey = (team_id: string) =>
"concurrency-limit-queue:" + team_id;
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 6ed5dbf03..cfb20152d 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -5,10 +5,12 @@ import {
getConcurrencyLimitActiveJobs,
getConcurrencyQueueJobsCount,
getCrawlConcurrencyLimitActiveJobs,
+ getTeamQueueLimit,
pushConcurrencyLimitActiveJob,
pushConcurrencyLimitedJob,
pushConcurrencyLimitedJobs,
pushCrawlConcurrencyLimitActiveJob,
+ QueueFullError,
} from "../lib/concurrency-limit";
import { logger as _logger } from "../lib/logger";
import { sendNotificationWithCustomDays } from "./notification/email_notification";
@@ -259,19 +261,20 @@ async function addScrapeJobRaw(
}
}
+ maxConcurrency =
+ (
+ await getACUCTeam(
+ webScraperOptions.team_id,
+ false,
+ true,
+ RateLimiterMode.Crawl,
+ )
+ )?.concurrency ?? 2;
+
if (concurrencyLimited === null) {
const now = Date.now();
- const maxConcurrency =
- (
- await getACUCTeam(
- webScraperOptions.team_id,
- false,
- true,
- RateLimiterMode.Crawl,
- )
- )?.concurrency ?? 2;
await cleanOldConcurrencyLimitEntries(webScraperOptions.team_id, now);
- const currentActiveConcurrency = (
+ currentActiveConcurrency = (
await getConcurrencyLimitActiveJobs(webScraperOptions.team_id, now)
).length;
concurrencyLimited =
@@ -280,13 +283,19 @@ async function addScrapeJobRaw(
}
if (concurrencyLimited === "yes" || concurrencyLimited === "yes-crawl") {
+ const concurrencyQueueJobs = await getConcurrencyQueueJobsCount(
+ webScraperOptions.team_id,
+ );
+
+ const queueLimit = getTeamQueueLimit(maxConcurrency);
+ if (concurrencyQueueJobs >= queueLimit) {
+ throw new QueueFullError(concurrencyQueueJobs, queueLimit);
+ }
+
if (concurrencyLimited === "yes") {
// Detect if they hit their concurrent limit
// If above by 2x, send them an email
// No need to 2x as if there are more than the max concurrency in the concurrency queue, it is already 2x
- const concurrencyQueueJobs = await getConcurrencyQueueJobsCount(
- webScraperOptions.team_id,
- );
if (concurrencyQueueJobs > maxConcurrency) {
// logger.info("Concurrency limited 2x (single) - ", "Concurrency queue jobs: ", concurrencyQueueJobs, "Max concurrency: ", maxConcurrency, "Team ID: ", webScraperOptions.team_id);
@@ -489,6 +498,14 @@ export async function addScrapeJobs(
addToCQ = jobsPotentiallyInCQ
.slice(countCanBeDirectlyAdded)
.concat(jobsForcedToCQ);
+
+ if (addToCQ.length > 0) {
+ const currentQueueSize = await getConcurrencyQueueJobsCount(teamId);
+ const queueLimit = getTeamQueueLimit(maxConcurrency);
+ if (currentQueueSize + addToCQ.length > queueLimit) {
+ throw new QueueFullError(currentQueueSize, queueLimit);
+ }
+ }
}
// equals 2x the max concurrency (only check for non-self-hosted)
diff --git a/apps/api/src/services/supabase.ts b/apps/api/src/services/supabase.ts
index d6a21229f..7e99131d9 100644
--- a/apps/api/src/services/supabase.ts
+++ b/apps/api/src/services/supabase.ts
@@ -5,7 +5,7 @@ import { logger } from "../lib/logger";
configDotenv();
/** PostgREST code when .single() returns 0 or >1 rows. Use to distinguish "no row" from real DB errors. */
-export const POSTGREST_NO_ROWS_CODE = "PGRST116";
+const POSTGREST_NO_ROWS_CODE = "PGRST116";
export function isPostgrestNoRowsError(
error: { code?: string } | null | undefined,