Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore(api): drop ACUC concurrency and rate_limits references (#4131)
firecrawl-spring[bot]3fded32f30bb
diff --git a/apps/api/src/controllers/__tests__/auth.test.ts b/apps/api/src/controllers/__tests__/auth.test.tsindex c888c016c..1be79f79a 100644--- a/apps/api/src/controllers/__tests__/auth.test.ts+++ b/apps/api/src/controllers/__tests__/auth.test.ts@@ -138,9 +138,7 @@ describe("authenticateUser", () => {api_key_id: 1,team_id: "team-1",org_id: "org-1",- rate_limits: { scrape: 10 },plan_priority: {},- concurrency: 2,flags: null,},]);@@ -182,9 +180,7 @@ describe("authenticateUser", () => {api_key_id: 1,team_id: "team-1",org_id: "org-1",- rate_limits: { crawl: 10 },plan_priority: {},- concurrency: 2,flags: null,},]);@@ -361,9 +357,7 @@ describe("authenticateUser", () => {api_key_id: 1,team_id: "team-2",org_id: "org-2",- rate_limits: { scrape: 10 },plan_priority: {},- concurrency: 2,flags: null,},]);diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 2c2c80b40..7540da00b 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -86,23 +86,10 @@ const mockPreviewACUC: (api_key_id: 0,team_id,org_id: "preview",- rate_limits: {- crawl: 2,- scrape: 10,- extract: 10,- search: 5,- map: 5,- preview: 5,- crawlStatus: 500,- extractStatus: 500,- extractAgentPreview: 1,- scrapeAgentPreview: 5,- },plan_priority: {bucketLimit: 25,planModifier: 0.1,},- concurrency: is_extract ? 200 : 2,flags: null,is_extract,});@@ -112,23 +99,10 @@ const mockACUC: () => AuthCreditUsageChunk = () => ({api_key_id: 0,team_id: "bypass",org_id: "bypass",- rate_limits: {- crawl: 99999999,- scrape: 99999999,- extract: 99999999,- search: 99999999,- map: 99999999,- preview: 99999999,- crawlStatus: 99999999,- extractStatus: 99999999,- extractAgentPreview: 99999999,- scrapeAgentPreview: 99999999,- },plan_priority: {bucketLimit: 25,planModifier: 0.1,},- concurrency: 99999999,flags: null,is_extract: false,});@@ -646,11 +620,11 @@ async function supaAuthenticateUser(}if (token == config.PREVIEW_TOKEN) {if (mode == RateLimiterMode.CrawlStatus) {- rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, null);+ rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus);} else if (mode == RateLimiterMode.ExtractStatus) {- rateLimiter = getRateLimiter(RateLimiterMode.ExtractStatus, null);+ rateLimiter = getRateLimiter(RateLimiterMode.ExtractStatus);} else {- rateLimiter = getRateLimiter(RateLimiterMode.Preview, null);+ rateLimiter = getRateLimiter(RateLimiterMode.Preview);}teamId = `preview_${iptoken}`;} else if (token.startsWith("fcmcp_")) {@@ -829,7 +803,6 @@ async function supaAuthenticateUser(// logger.error(`Rate limit exceeded: ${rateLimiterRes}`, {// teamId,// mode,- // rateLimits: chunk?.rate_limits,// rateLimiterRes,// });diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 0e0788fe5..344049f3a 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1262,26 +1262,6 @@ export type AuthCreditUsageChunk = {bucketLimit: number;planModifier: number;};- rate_limits: {- crawl: number;- scrape: number;- search: number;- map: number;- extract: number;- preview: number;- crawlStatus: number;- extractStatus: number;- extractAgentPreview?: number;- scrapeAgentPreview?: number;- browser?: number;- browserExecute?: number;- browserReplay?: number;- account?: number;- supportAsk?: number;- supportDocsSearch?: number;- research?: number;- };- concurrency: number;flags: TeamFlags;// appended on JS-sidediff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.tsindex 8dac08c06..d3981c865 100644--- a/apps/api/src/services/rate-limiter.ts+++ b/apps/api/src/services/rate-limiter.ts@@ -2,7 +2,6 @@ import { RateLimiterRedis } from "rate-limiter-flexible";import { config } from "../config";import { RateLimiterMode } from "../types";import Redis from "ioredis";-import type { AuthCreditUsageChunk } from "../controllers/v1/types";export const redisRateLimitClient = new Redis(config.REDIS_RATE_LIMIT_URL!, {enableAutoPipelining: true,@@ -16,7 +15,9 @@ const createRateLimiter = (keyPrefix, points) =>duration: 60, // Duration in seconds});-const fallbackRateLimits: AuthCreditUsageChunk["rate_limits"] = {+// Keyed by RateLimiterMode so a new mode fails to compile until it has a+// fallback here, rather than silently taking the 500 default below.+const fallbackRateLimits: Record<RateLimiterMode, number> = {crawl: 15,scrape: 100,search: 100,@@ -59,15 +60,12 @@ const BASE_RATE_LIMITS: Partial<Record<RateLimiterMode, number>> = {};/**- * Builds the per-minute rate limiter for a mode from a `rate_limits` map (or the- * static fallback table). Used for the preview token, which has no Autumn- * entity; authenticated teams use getAutumnRateLimiter.+ * Builds the per-minute rate limiter for a mode from the static fallback table.+ * Used for the preview token, which has no Autumn entity; authenticated teams+ * use getAutumnRateLimiter.*/-export function getRateLimiter(- mode: RateLimiterMode,- rate_limits: AuthCreditUsageChunk["rate_limits"] | null,-): RateLimiterRedis {- let rateLimit = rate_limits?.[mode] ?? fallbackRateLimits?.[mode] ?? 500;+export function getRateLimiter(mode: RateLimiterMode): RateLimiterRedis {+ let rateLimit = fallbackRateLimits?.[mode] ?? 500;if (mode === RateLimiterMode.Search || mode === RateLimiterMode.Scrape) {// TEMP: Mogery