Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
refactor(api): remove the higher-rate-limits experiment (#5460)
Maxd36ae2f8ba3e
diff --git a/apps/api/src/controllers/__tests__/auth.test.ts b/apps/api/src/controllers/__tests__/auth.test.tsindex 5143edb6d..4f51d1d5b 100644--- a/apps/api/src/controllers/__tests__/auth.test.ts+++ b/apps/api/src/controllers/__tests__/auth.test.ts@@ -27,7 +27,6 @@ import { trackKeylessPromptShown } from "../../lib/keyless-prompt-analytics";import { mintWorldIdCredential, worldIdSubjectHash } from "../../lib/world-id";import { db } from "../../db/connection";import { autumnService } from "../../services/autumn/autumn.service";-import { getRateLimitExperimentVariant } from "../../services/rate-limit-experiment";vi.mock("../../services/queue-service", () => ({getRedisConnection: vi.fn(() => ({@@ -110,10 +109,6 @@ vi.mock("../../services/autumn/autumn.service", () => ({},}));-vi.mock("../../services/rate-limit-experiment", () => ({- getRateLimitExperimentVariant: vi.fn(),-}));-vi.mock("../../services/agent-sponsor", () => ({getAgentSponsorStatus: vi.fn(),}));@@ -1108,88 +1103,6 @@ describe("authenticateUser", () => {);});- describe("rate-limit experiment", () => {- const request = {- headers: {- authorization: "Bearer 00000000-0000-4000-8000-000000000000",- },- socket: { remoteAddress: "127.0.0.1" },- };- const chunkWith = (extra: Record<string, unknown> = {}) => [- {- api_key: "00000000-0000-4000-8000-000000000000",- api_key_id: 1,- team_id: "team-1",- org_id: "org-1",- flags: {},- ...extra,- },- ];-- beforeEach(() => {- config.USE_DB_AUTHENTICATION = true;- vi.mocked(getValue).mockResolvedValue(null);- vi.mocked(authCreditUsageChunk).mockResolvedValue(chunkWith());- vi.mocked(redlock.using).mockImplementation(- async (_keys, _ttl, _options, fn) => fn({ aborted: false } as never),- );- mockMultiplier(1);- });-- it("gives a treatment org the raised limit", async () => {- vi.mocked(getRateLimitExperimentVariant).mockResolvedValue("treatment");-- const auth = await authenticateUser(request, {}, RateLimiterMode.Scrape);-- expect(auth.success).toBe(true);- expect(getRateLimitExperimentVariant).toHaveBeenCalledExactlyOnceWith({- orgId: "org-1",- teamId: "team-1",- apiKeyId: 1,- rateLimitMultiplier: 1,- isPaidPlan: false,- });- expect(getAutumnRateLimiter).toHaveBeenLastCalledWith(- RateLimiterMode.Scrape,- 1,- {},- "treatment",- );- });-- it("keeps the current limit for a control or unenrolled org", async () => {- for (const variant of ["control", undefined] as const) {- vi.mocked(getAutumnRateLimiter).mockClear();- vi.mocked(getRateLimitExperimentVariant).mockResolvedValue(variant);-- await authenticateUser(request, {}, RateLimiterMode.Scrape);-- expect(getAutumnRateLimiter).toHaveBeenCalledExactlyOnceWith(- RateLimiterMode.Scrape,- 1,- {},- );- }- });-- it("does not ask for a variant where it cannot change the limit", async () => {- await authenticateUser(request, {}, RateLimiterMode.Map);-- expect(getRateLimitExperimentVariant).not.toHaveBeenCalled();- });-- it("does not enroll a request denied by the auth checks", async () => {- vi.mocked(authCreditUsageChunk).mockResolvedValue(- chunkWith({ is_banned: true }),- );-- const auth = await authenticateUser(request, {}, RateLimiterMode.Scrape);-- expect(auth.success).toBe(false);- expect(getRateLimitExperimentVariant).not.toHaveBeenCalled();- });- });-describe("hosted MCP rate-limit minimum", () => {const API_KEY = "Bearer 00000000-0000-4000-8000-000000000000";const requestWith = (@@ -1332,31 +1245,6 @@ describe("authenticateUser", () => {expect(planConsume).toHaveBeenCalledExactlyOnceWith("team-1");});- it("does not enroll the org in the experiment through a hosted MCP call", async () => {- await authenticateUser(- requestWith(`Bearer ${signDelegation()}`),- {},- RateLimiterMode.Scrape,- HOSTED_MCP_ROUTE,- );-- expect(getRateLimitExperimentVariant).not.toHaveBeenCalled();- expect(hostedMcpConsume).toHaveBeenCalledExactlyOnceWith("team-1");- });-- it("runs the experiment when the hosted MCP minimum does not apply", async () => {- vi.mocked(getHostedMcpRateLimiter).mockReturnValue(undefined);-- await authenticateUser(- requestWith(`Bearer ${signDelegation()}`),- {},- RateLimiterMode.Scrape,- HOSTED_MCP_ROUTE,- );-- expect(getRateLimitExperimentVariant).toHaveBeenCalledOnce();- });-it.each([["", {}],[@@ -1646,10 +1534,6 @@ describe("authenticateUser", () => {HOBBY_RATE_LIMIT_MULTIPLIER,flags,);- // The experiment records the team's plan, not the floor.- expect(getRateLimitExperimentVariant).toHaveBeenCalledWith(- expect.objectContaining({ rateLimitMultiplier: 1, isPaidPlan: false }),- );});it("leaves a paid plan's multiplier alone for a trusted agent request", async () => {diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex ff0c3b1f8..ae0171aaa 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -13,9 +13,7 @@ import {getHostedMcpRateLimiter,getRateLimitOverride,HOBBY_RATE_LIMIT_MULTIPLIER,- rateLimitExperimentApplies,} from "../services/rate-limiter";-import { getRateLimitExperimentVariant } from "../services/rate-limit-experiment";import { isTrustedAgentInteropRequest } from "../lib/agent-interop";import {consumeKeylessRequest,@@ -1141,14 +1139,7 @@ async function supaAuthenticateUser(}}- // The rate-limit experiment runs after the ban and restriction checks, so- // only requests that reach the limiter enroll an org or count as exposed.if (chunk !== null) {- const multiplier = authenticatedRateLimitMultiplier(- chunk,- mode,- minRateMultiplier,- );// Calls the hosted MCP makes on routes that opt in (every v2 route) get a// raised per-minute minimum, and scrape calls a raised concurrency// minimum. Two signals prove the hosted MCP made the call: a verified@@ -1165,9 +1156,8 @@ async function supaAuthenticateUser(req.headers["x-firecrawl-keyless-secret"],config.KEYLESS_PROXY_SECRET,)));- // The tiers follow the team's own plan multiplier, not `multiplier`,- // which trusted agent traffic floors at hobby, so a free team never gets- // the hobby tier.+ // The tiers follow the team's own plan multiplier, not the one trusted+ // agent traffic floors at hobby, so a free team never gets the hobby tier.const hostedMcpLimiter = hostedMcpRaise? getHostedMcpRateLimiter(mode, chunk.rate_limit_multiplier, chunk.flags): undefined;@@ -1184,26 +1174,7 @@ async function supaAuthenticateUser(chunk = { ...chunk, concurrency_limit: hostedMcpConcurrency };}if (hostedMcpLimiter) {- // At or above the treatment minimum on every plan it applies to, so the- // experiment cannot change this call's limit and does not enroll the- // org through it.rateLimiter = hostedMcpLimiter;- } else if (- rateLimitExperimentApplies(mode, multiplier, chunk.flags) &&- (await getRateLimitExperimentVariant({- orgId: chunk.org_id,- teamId: chunk.team_id,- apiKeyId: chunk.api_key_id_text ?? chunk.api_key_id,- rateLimitMultiplier: chunk.rate_limit_multiplier,- isPaidPlan: chunk.is_paid_plan,- })) === "treatment"- ) {- rateLimiter = getAutumnRateLimiter(- mode,- multiplier,- chunk.flags,- "treatment",- );}}diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.tsindex 5b3a85673..954ddaf02 100644--- a/apps/api/src/lib/zdr-helpers.ts+++ b/apps/api/src/lib/zdr-helpers.ts@@ -89,7 +89,7 @@ export function getCustomRobotsAgent(}/** The agent ZDR result retentions an org can be given, in minutes. */-export const AGENT_ZDR_RETENTION_TIERS_MINUTES = [15, 30, 60, 360, 1440];+const AGENT_ZDR_RETENTION_TIERS_MINUTES = [15, 30, 60, 360, 1440];/** The agent service's ZDR result retention when the team flag is unset. */export const AGENT_ZDR_DEFAULT_RETENTION_MINUTES = 24 * 60;diff --git a/apps/api/src/services/posthog-capture.test.ts b/apps/api/src/services/posthog-capture.test.tsindex 64364b7c6..5f02ac2be 100644--- a/apps/api/src/services/posthog-capture.test.ts+++ b/apps/api/src/services/posthog-capture.test.ts@@ -64,133 +64,3 @@ describe("capturePostHog", () => {expect(fetchMock).not.toHaveBeenCalled();});});--describe("evaluatePostHogFlag", () => {- const flagsResponse = (body: unknown) =>- vi.fn().mockResolvedValue(new Response(JSON.stringify(body)));-- it("returns the variant and asks for that flag with the groups", async () => {- const fetchMock = flagsResponse({- flags: { "my-flag": { enabled: true, variant: "treatment" } },- errorsWhileComputingFlags: false,- });- vi.stubGlobal("fetch", fetchMock);- const { evaluatePostHogFlag } = await load("phc_test");-- await expect(- evaluatePostHogFlag("my-flag", "d", { company: "org-1" }),- ).resolves.toEqual({ status: "resolved", variant: "treatment" });- expect(fetchMock).toHaveBeenCalledWith(- "https://ph.example.com/flags/?v=2",- expect.objectContaining({- method: "POST",- body: JSON.stringify({- api_key: "phc_test",- distinct_id: "d",- groups: { company: "org-1" },- flag_keys_to_evaluate: ["my-flag"],- }),- }),- );- });-- it.each([401, 404])(- "is disabled when PostHog rejects the request with %i",- async status => {- vi.stubGlobal(- "fetch",- vi.fn().mockResolvedValue(new Response("no", { status })),- );- const { evaluatePostHogFlag } = await load("phc_test");-- await expect(evaluatePostHogFlag("my-flag", "d")).resolves.toEqual({- status: "disabled",- });- },- );-- it.each([- ["an unknown flag", { flags: {} }],- ["a flag that is off", { flags: { "my-flag": { enabled: false } } }],- [- "a boolean flag",- { flags: { "my-flag": { enabled: true, variant: null } } },- ],- ])("is disabled for %s", async (_label, body) => {- vi.stubGlobal("fetch", flagsResponse(body));- const { evaluatePostHogFlag } = await load("phc_test");-- await expect(evaluatePostHogFlag("my-flag", "d")).resolves.toEqual({- status: "disabled",- });- });-- it.each([- ["a 503 answer", () => new Response("no", { status: 503 })],- ["a 429 answer", () => new Response("no", { status: 429 })],- [- "errors while computing flags",- () =>- new Response(- JSON.stringify({ flags: {}, errorsWhileComputingFlags: true }),- ),- ],- [- "a quota limit",- () =>- new Response(- JSON.stringify({ flags: {}, quotaLimited: ["feature_flags"] }),- ),- ],- ])("is retryable on %s", async (_label, response) => {- vi.stubGlobal("fetch", vi.fn().mockResolvedValue(response()));- const { evaluatePostHogFlag } = await load("phc_test");-- await expect(evaluatePostHogFlag("my-flag", "d")).resolves.toEqual({- status: "retryable",- });- });-- it("is retryable, without rejecting, on a network error", async () => {- vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new TypeError("failed")));- const { evaluatePostHogFlag } = await load("phc_test");-- await expect(evaluatePostHogFlag("my-flag", "d")).resolves.toEqual({- status: "retryable",- });- });-- it("gives up after 500 ms and is retryable", async () => {- // Answers only when the request is aborted, like a PostHog that hangs.- vi.stubGlobal(- "fetch",- vi.fn(- (_url: string, init: RequestInit) =>- new Promise((_, reject) =>- init.signal!.addEventListener("abort", () =>- reject(init.signal!.reason),- ),- ),- ),- );- const { evaluatePostHogFlag } = await load("phc_test");-- const start = Date.now();- await expect(evaluatePostHogFlag("my-flag", "d")).resolves.toEqual({- status: "retryable",- });- expect(Date.now() - start).toBeGreaterThanOrEqual(450);- expect(Date.now() - start).toBeLessThan(1500);- });-- it("asks nothing and is disabled with no key", async () => {- const fetchMock = vi.fn();- vi.stubGlobal("fetch", fetchMock);- const { evaluatePostHogFlag } = await load("");-- await expect(evaluatePostHogFlag("my-flag", "d")).resolves.toEqual({- status: "disabled",- });- expect(fetchMock).not.toHaveBeenCalled();- });-});diff --git a/apps/api/src/services/posthog-capture.ts b/apps/api/src/services/posthog-capture.tsindex 10bff754f..fad00248c 100644--- a/apps/api/src/services/posthog-capture.ts+++ b/apps/api/src/services/posthog-capture.ts@@ -1,15 +1,14 @@import { logger as _logger } from "../lib/logger";/**- * Lightweight, dependency-free PostHog capture and flag evaluation for the API.+ * Lightweight, dependency-free PostHog capture for the API.*- * The API has no PostHog SDK wired up, so we POST directly to the capture and- * flags endpoints. Everything here is best-effort: a missing key or a network- * error must never affect request handling.+ * The API has no PostHog SDK wired up, so we POST directly to the capture+ * endpoint. Everything here is best-effort and fire-and-forget: a missing key+ * or a network error must never affect request handling.** Configure via env:- * POSTHOG_API_KEY: project API key (if unset, capture and flag reads are- * no-ops)+ * POSTHOG_API_KEY: project API key (if unset, capture is a no-op)* POSTHOG_HOST: ingestion host (defaults to https://us.i.posthog.com)*/const POSTHOG_API_KEY = process.env.POSTHOG_API_KEY;@@ -64,76 +63,3 @@ export async function capturePostHog(}return false;}--// Bounds the wait for a flag read, matching the server-side flag reads in-// firecrawl-web: a slow PostHog delays a request by at most this much.-const FLAG_TIMEOUT_MS = 500;--export type PostHogFlagResult =- | { status: "resolved"; variant: string }- /** Off, unknown, or not matched for this caller: a stable non-assignment. */- | { status: "disabled" }- /** A timeout, error or quota limit: transient, so callers retry later. */- | { status: "retryable" };--/**- * Evaluate one multivariate flag with PostHog's `/flags` endpoint. `groups`- * carries the group keys of a group-aggregated flag, e.g. `{ company: orgId }`- * for an org-level experiment. Never rejects.- */-export async function evaluatePostHogFlag(- key: string,- distinctId: string,- groups: Record<string, string> = {},-): Promise<PostHogFlagResult> {- if (!POSTHOG_API_KEY) return { status: "disabled" };-- try {- const response = await fetch(- `${POSTHOG_HOST.replace(/\/$/, "")}/flags/?v=2`,- {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify({- api_key: POSTHOG_API_KEY,- distinct_id: distinctId,- groups,- flag_keys_to_evaluate: [key],- }),- signal: AbortSignal.timeout(FLAG_TIMEOUT_MS),- },- );- if (!response.ok) {- _logger.debug("PostHog flag evaluation rejected", {- module: "posthog",- key,- status: response.status,- });- // Only rate limiting and server errors pass; any other rejection, e.g.- // a wrong key or host, enrolls nobody like a flag that is off.- return response.status === 429 || response.status >= 500- ? { status: "retryable" }- : { status: "disabled" };- }- const body = (await response.json()) as {- flags?: Record<string, { enabled?: boolean; variant?: unknown }>;- errorsWhileComputingFlags?: boolean;- quotaLimited?: string[] | null;- };- if (body.errorsWhileComputingFlags || body.quotaLimited?.length) {- return { status: "retryable" };- }- const flag = body.flags?.[key];- if (flag?.enabled && typeof flag.variant === "string") {- return { status: "resolved", variant: flag.variant };- }- return { status: "disabled" };- } catch (error) {- _logger.debug("PostHog flag evaluation failed", {- module: "posthog",- key,- error,- });- return { status: "retryable" };- }-}diff --git a/apps/api/src/services/posthog.ts b/apps/api/src/services/posthog.tsindex 9dce8d1cf..350702bf0 100644--- a/apps/api/src/services/posthog.ts+++ b/apps/api/src/services/posthog.ts@@ -50,7 +50,7 @@ function originToSurface(** Only called on rare, deduplicated events, so the extra read is cheap.*/-export async function resolveDistinctId(+async function resolveDistinctId(teamId: string,apiKeyId?: string | number | null,): Promise<string> {diff --git a/apps/api/src/services/rate-limit-experiment.test.ts b/apps/api/src/services/rate-limit-experiment.test.tsdeleted file mode 100644index b228c46af..000000000--- a/apps/api/src/services/rate-limit-experiment.test.ts+++ /dev/null@@ -1,349 +0,0 @@-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";--const mocks = vi.hoisted(() => ({- /** What the Redis mock holds, so a test sees stored assignments and markers. */- store: new Map<string, string>(),- get: vi.fn(),- set: vi.fn(),- del: vi.fn(),- expire: vi.fn(),- evaluate: vi.fn(),- capture: vi.fn(),- enabled: vi.fn(),- distinctId: vi.fn(),-}));--vi.mock("./redis", () => ({- redisEvictConnection: {- get: mocks.get,- set: mocks.set,- del: mocks.del,- expire: mocks.expire,- },-}));-vi.mock("./posthog-capture", () => ({- evaluatePostHogFlag: mocks.evaluate,- capturePostHog: mocks.capture,- isPostHogCaptureEnabled: mocks.enabled,-}));-vi.mock("./posthog", () => ({ resolveDistinctId: mocks.distinctId }));--import { getRateLimitExperimentVariant } from "./rate-limit-experiment";--/** Let the fire-and-forget tasks run. */-const settle = () => new Promise(resolve => setImmediate(resolve));--const caller = (orgId: string) => ({- orgId,- teamId: "team-1",- apiKeyId: 7,- rateLimitMultiplier: 10,- isPaidPlan: true,-});-const assignmentKey = (orgId: string) =>- `firecrawl:rate_limit_experiment:${orgId}`;--beforeEach(() => {- mocks.store.clear();- mocks.enabled.mockReturnValue(true);- mocks.get.mockImplementation(async (key: string) => mocks.store.get(key));- mocks.set.mockImplementation(async (key: string, value: string, ...args) => {- if (args.includes("NX") && mocks.store.has(key)) return null;- mocks.store.set(key, value);- return "OK";- });- mocks.del.mockImplementation(async (key: string) =>- mocks.store.delete(key) ? 1 : 0,- );- mocks.evaluate.mockResolvedValue({- status: "resolved",- variant: "treatment",- });- mocks.expire.mockResolvedValue(1);- mocks.capture.mockResolvedValue(true);- mocks.distinctId.mockResolvedValue("[email protected]");-});--afterEach(async () => {- // A test must not leave an exposure send running into the next one.- await settle();- vi.useRealTimers();- vi.clearAllMocks();-});--// The module keeps per-process caches for the life of the process, so each-// test uses its own org id.-describe("getRateLimitExperimentVariant", () => {- it("asks PostHog for the org's variant once, stores it, and sends the exposure with the plan", async () => {- await expect(- getRateLimitExperimentVariant(caller("org-new")),- ).resolves.toBe("treatment");- await settle();-- // The flag buckets on the org, so the read needs no DB lookup.- expect(mocks.evaluate).toHaveBeenCalledExactlyOnceWith(- "higher-rate-limits",- "org-new",- { company: "org-new" },- );- expect(mocks.distinctId).toHaveBeenCalledExactlyOnceWith("team-1", 7);- expect(JSON.parse(mocks.store.get(assignmentKey("org-new"))!)).toEqual({- variant: "treatment",- checkedAt: expect.any(Number),- });- expect(mocks.capture).toHaveBeenCalledExactlyOnceWith(- "$feature_flag_called",- "[email protected]",- {- $feature_flag: "higher-rate-limits",- $feature_flag_response: "treatment",- team_id: "team-1",- rate_limit_multiplier: 10,- is_paid_plan: true,- $groups: { company: "org-new", team: "team-1" },- },- );-- // The marker is short-lived until PostHog accepts the event.- expect(mocks.set).toHaveBeenCalledWith(- "firecrawl:rate_limit_experiment_exposure:org-new:treatment",- "1",- "EX",- 5 * 60,- "NX",- );- expect(mocks.expire).toHaveBeenCalledExactlyOnceWith(- "firecrawl:rate_limit_experiment_exposure:org-new:treatment",- 30 * 24 * 60 * 60,- );-- await getRateLimitExperimentVariant(caller("org-new"));- await settle();- expect(mocks.evaluate).toHaveBeenCalledTimes(1);- expect(mocks.capture).toHaveBeenCalledTimes(1);- });-- it("reads a fresh stored assignment without asking PostHog", async () => {- mocks.store.set(- assignmentKey("org-stored"),- JSON.stringify({ variant: "control", checkedAt: Date.now() }),- );-- await expect(- getRateLimitExperimentVariant(caller("org-stored")),- ).resolves.toBe("control");- await settle();- expect(mocks.evaluate).not.toHaveBeenCalled();- });-- it("treats a malformed stored assignment as missing", async () => {- mocks.store.set(- assignmentKey("org-malformed"),- JSON.stringify({ variant: "test", checkedAt: Date.now() }),- );-- await expect(- getRateLimitExperimentVariant(caller("org-malformed")),- ).resolves.toBe("treatment");- await settle();- expect(mocks.evaluate).toHaveBeenCalledTimes(1);- });-- it("serves a stale assignment while it re-reads PostHog in the background", async () => {- mocks.store.set(- assignmentKey("org-stale"),- JSON.stringify({ variant: "treatment", checkedAt: 0 }),- );- // Paused in PostHog since the org was assigned.- mocks.evaluate.mockResolvedValue({ status: "disabled" });-- await expect(- getRateLimitExperimentVariant(caller("org-stale")),- ).resolves.toBe("treatment");- await settle();-- expect(mocks.evaluate).toHaveBeenCalledTimes(1);- expect(JSON.parse(mocks.store.get(assignmentKey("org-stale"))!)).toEqual({- variant: null,- checkedAt: expect.any(Number),- });- await expect(- getRateLimitExperimentVariant(caller("org-stale")),- ).resolves.toBeUndefined();- });-- it.each([- ["the flag is off or paused", "org-off", { status: "disabled" }],- [- "PostHog serves an unknown variant",- "org-unknown",- { status: "resolved", variant: "x" },- ],- ])("enrolls nobody when %s, and stores that", async (_label, org, result) => {- mocks.evaluate.mockResolvedValue(result);-- await expect(getRateLimitExperimentVariant(caller(org))).resolves.toBe(- undefined,- );- await settle();-- expect(mocks.capture).not.toHaveBeenCalled();- expect(JSON.parse(mocks.store.get(assignmentKey(org))!).variant).toBe(null);- });-- it("keeps the current limits on a transient failure and retries a minute later", async () => {- vi.useFakeTimers({ toFake: ["Date"] });- mocks.evaluate.mockResolvedValueOnce({ status: "retryable" });-- await expect(- getRateLimitExperimentVariant(caller("org-flaky")),- ).resolves.toBeUndefined();- expect(mocks.store.has(assignmentKey("org-flaky"))).toBe(false);-- await getRateLimitExperimentVariant(caller("org-flaky"));- expect(mocks.evaluate).toHaveBeenCalledTimes(1);-- vi.advanceTimersByTime(61_000);- await expect(- getRateLimitExperimentVariant(caller("org-flaky")),- ).resolves.toBe("treatment");- await settle();- expect(mocks.evaluate).toHaveBeenCalledTimes(2);- });-- it("asks PostHog once for concurrent first requests", async () => {- await Promise.all([- getRateLimitExperimentVariant(caller("org-burst")),- getRateLimitExperimentVariant(caller("org-burst")),- getRateLimitExperimentVariant(caller("org-burst")),- ]);- await settle();-- expect(mocks.evaluate).toHaveBeenCalledTimes(1);- expect(mocks.capture).toHaveBeenCalledTimes(1);- });-- it("keeps an accepted exposure reported when extending its marker fails", async () => {- mocks.expire.mockRejectedValueOnce(new Error("redis down"));-- await getRateLimitExperimentVariant(caller("org-expire"));- await settle();- await getRateLimitExperimentVariant(caller("org-expire"));- await settle();-- expect(mocks.capture).toHaveBeenCalledTimes(1);- });-- it("does not resend an exposure another process holds, but checks again once its marker could expire", async () => {- vi.useFakeTimers({ toFake: ["Date"] });- const marker =- "firecrawl:rate_limit_experiment_exposure:org-elsewhere:treatment";- mocks.store.set(marker, "1");-- await getRateLimitExperimentVariant(caller("org-elsewhere"));- await settle();- await getRateLimitExperimentVariant(caller("org-elsewhere"));- await settle();- expect(mocks.set).toHaveBeenCalledTimes(2); // the assignment, one NX try- expect(mocks.capture).not.toHaveBeenCalled();-- // That process's send failed and released the marker.- mocks.store.delete(marker);- vi.advanceTimersByTime(5 * 60 * 1000 + 1);- await getRateLimitExperimentVariant(caller("org-elsewhere"));- await settle();- expect(mocks.capture).toHaveBeenCalledTimes(1);- });-- it("serves the variant PostHog assigned even when storing it fails", async () => {- mocks.set.mockRejectedValueOnce(new Error("redis down"));-- await expect(- getRateLimitExperimentVariant(caller("org-unstored")),- ).resolves.toBe("treatment");- await settle();- await expect(- getRateLimitExperimentVariant(caller("org-unstored")),- ).resolves.toBe("treatment");- expect(mocks.evaluate).toHaveBeenCalledTimes(1);- });-- it("does not hold the request while Redis stalls storing the assignment", async () => {- mocks.set.mockReturnValueOnce(new Promise(() => {}));-- const start = Date.now();- await expect(- getRateLimitExperimentVariant(caller("org-stalled-store")),- ).resolves.toBe("treatment");- expect(Date.now() - start).toBeLessThan(400);- });-- it("keeps the current limits while Redis stalls on a first read, and asks PostHog in the background", async () => {- mocks.get.mockReturnValueOnce(new Promise(() => {}));-- const start = Date.now();- await expect(- getRateLimitExperimentVariant(caller("org-stalled-read")),- ).resolves.toBeUndefined();- expect(Date.now() - start).toBeLessThan(400);- await settle();-- expect(mocks.evaluate).toHaveBeenCalledTimes(1);- await expect(- getRateLimitExperimentVariant(caller("org-stalled-read")),- ).resolves.toBe("treatment");- });-- it("keeps an org's last variant while Redis stalls, even if PostHog fails too", async () => {- vi.useFakeTimers({ toFake: ["Date"] });- mocks.store.set(- assignmentKey("org-stalled-known"),- JSON.stringify({ variant: "treatment", checkedAt: Date.now() }),- );- await getRateLimitExperimentVariant(caller("org-stalled-known"));- await settle();-- vi.advanceTimersByTime(61_000);- mocks.get.mockReturnValueOnce(new Promise(() => {}));- mocks.evaluate.mockResolvedValueOnce({ status: "retryable" });- await expect(- getRateLimitExperimentVariant(caller("org-stalled-known")),- ).resolves.toBe("treatment");- await settle();- expect(mocks.evaluate).toHaveBeenCalledTimes(1);- });-- it("releases the exposure marker when PostHog rejects it, and retries after a wait", async () => {- vi.useFakeTimers({ toFake: ["Date"] });- mocks.capture.mockResolvedValueOnce(false);- mocks.store.set(- assignmentKey("org-retry"),- JSON.stringify({ variant: "control", checkedAt: Date.now() }),- );- const marker = "firecrawl:rate_limit_experiment_exposure:org-retry:control";-- await getRateLimitExperimentVariant(caller("org-retry"));- await settle();- expect(mocks.store.has(marker)).toBe(false);- await getRateLimitExperimentVariant(caller("org-retry"));- await settle();- expect(mocks.capture).toHaveBeenCalledTimes(1);-- vi.advanceTimersByTime(5 * 60 * 1000 + 1);- await getRateLimitExperimentVariant(caller("org-retry"));- await settle();- expect(mocks.capture).toHaveBeenCalledTimes(2);- expect(mocks.expire).toHaveBeenCalledTimes(1);- expect(mocks.store.has(marker)).toBe(true);- });-- it("does nothing while PostHog is off", async () => {- mocks.enabled.mockReturnValue(false);-- await expect(- getRateLimitExperimentVariant(caller("org-no-posthog")),- ).resolves.toBeUndefined();- expect(mocks.evaluate).not.toHaveBeenCalled();- expect(mocks.get).not.toHaveBeenCalled();- });-});diff --git a/apps/api/src/services/rate-limit-experiment.ts b/apps/api/src/services/rate-limit-experiment.tsdeleted file mode 100644index 36559d7f8..000000000--- a/apps/api/src/services/rate-limit-experiment.ts+++ /dev/null@@ -1,334 +0,0 @@-import { logger as _logger } from "../lib/logger";-import { redisEvictConnection } from "./redis";-import {- capturePostHog,- evaluatePostHogFlag,- isPostHogCaptureEnabled,-} from "./posthog-capture";-import { resolveDistinctId } from "./posthog";-import type { RateLimitExperimentVariant } from "./rate-limiter";--/**- * PostHog experiment for the per-minute minimums in services/rate-limiter.ts.- * The flag is org-level (aggregated on the `company` group, key = org id) with- * `control` and `treatment` variants and no targeting conditions. PostHog- * assigns the variant; eligibility (a limit the variant can change) lives in- * code, so the flag is only read for eligible requests.- */-const RATE_LIMIT_EXPERIMENT_FLAG = "higher-rate-limits";--const VARIANTS: ReadonlySet<string> = new Set(["control", "treatment"]);--/**- * How long a stored assignment is trusted. After that it still serves while- * it is re-read in the background, so pausing or ending the experiment in- * PostHog reaches every org within about this long, and only a lookup with- * no stored answer waits on PostHog.- */-const ASSIGNMENT_FRESH_MS = 60 * 60 * 1000;-const ASSIGNMENT_TTL_SECONDS = 7 * 24 * 60 * 60;--// Per-process copy of the stored assignment, so most requests skip Redis. A-// transient PostHog failure is held here too, so it is retried a minute later-// rather than on every request.-const LOCAL_TTL_MS = 60 * 1000;-const LOCAL_MAX_ENTRIES = 100_000;--// The stored assignment is read in the auth path, so a read that takes longer-// treats Redis as unavailable rather than holding the request.-const REDIS_READ_TIMEOUT_MS = 100;--// An exposure marker is short-lived until PostHog accepts the event, so one a-// failed send leaves behind expires and the exposure is retried. An expired-// marker only resends an exposure PostHog already counts once per person and-// variant.-const EXPOSURE_PENDING_TTL_SECONDS = 5 * 60;-const EXPOSURE_TTL_SECONDS = 30 * 24 * 60 * 60;--type Assignment = {- /** Null when PostHog assigns no variant: flag off, paused or ended. */- variant: RateLimitExperimentVariant | null;- checkedAt: number;-};--type ExperimentCaller = {- orgId: string;- teamId: string;- /** The exact decimal id when known, as bigint ids can pass the safe range. */- apiKeyId?: string | number | null;- /**- * The team's plan when the exposure is sent. An org's first exposure, sent- * when it enrolls, so carries the plan it started on.- */- rateLimitMultiplier: number;- isPaidPlan: boolean;-};--const localAssignments = new Map<- string,- { assignment: Assignment | undefined; until: number }->();-const pendingAssignments = new Map<string, Promise<Assignment | undefined>>();-// Exposures not to send again before the time stored here: forever once this-// process sent one, for a pending marker's lifetime after another process held-// the marker or PostHog rejected the event.-const skipExposureUntil = new Map<string, number>();-const pendingExposures = new Set<string>();--const assignmentKey = (orgId: string) =>- `firecrawl:rate_limit_experiment:${orgId}`;--function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {- let timer: NodeJS.Timeout | undefined;- return Promise.race([- promise,- new Promise<never>((_, reject) => {- timer = setTimeout(- () => reject(new Error("Redis did not answer in time")),- ms,- );- timer.unref?.();- }),- ]).finally(() => clearTimeout(timer));-}--function setBounded<V>(map: Map<string, V>, key: string, value: V): void {- if (map.size >= LOCAL_MAX_ENTRIES) map.clear();- map.set(key, value);-}--// Redis did not answer, so whether the org has a stored assignment is unknown.-const STORE_UNAVAILABLE = Symbol("store-unavailable");--async function readAssignment(- orgId: string,-): Promise<Assignment | undefined | typeof STORE_UNAVAILABLE> {- let raw: string | null;- try {- raw = await withTimeout(- redisEvictConnection.get(assignmentKey(orgId)),- REDIS_READ_TIMEOUT_MS,- );- } catch {- return STORE_UNAVAILABLE;- }- if (!raw) return undefined;- try {- const parsed = JSON.parse(raw) as Partial<Assignment> | null;- // Anything else, e.g. a variant this code no longer knows, is a miss.- if (- typeof parsed?.checkedAt !== "number" ||- (parsed.variant !== null &&- (typeof parsed.variant !== "string" || !VARIANTS.has(parsed.variant)))- ) {- return undefined;- }- return { variant: parsed.variant, checkedAt: parsed.checkedAt };- } catch {- return undefined;- }-}--/**- * Read the org's variant from PostHog and store it. Undefined on a transient- * failure, which is not stored. Concurrent calls for an org share one read.- */-function fetchAssignment(- caller: ExperimentCaller,-): Promise<Assignment | undefined> {- const pending = pendingAssignments.get(caller.orgId);- if (pending) return pending;-- const promise = (async () => {- try {- // The flag buckets on the company group, so the distinct id does not- // decide the variant, and the org id keeps a DB read out of this path.- const result = await evaluatePostHogFlag(- RATE_LIMIT_EXPERIMENT_FLAG,- caller.orgId,- { company: caller.orgId },- );- if (result.status === "retryable") return undefined;- // A variant key PostHog serves but code doesn't know (e.g. mistyped in- // the UI) enrolls nobody, like a flag that is off.- const variant =- result.status === "resolved" && VARIANTS.has(result.variant)- ? (result.variant as RateLimitExperimentVariant)- : null;- const assignment: Assignment = { variant, checkedAt: Date.now() };- // Not awaited, so a stalled Redis never holds the request. A failed- // store still serves the answer; the per-process copy holds it and the- // next lookup after that asks PostHog again.- void redisEvictConnection- .set(- assignmentKey(caller.orgId),- JSON.stringify(assignment),- "EX",- ASSIGNMENT_TTL_SECONDS,- )- .catch(error => {- _logger.debug("Rate-limit experiment assignment not stored", {- module: "rate-limit-experiment",- orgId: caller.orgId,- error,- });- });- return assignment;- } catch (error) {- _logger.debug("Rate-limit experiment assignment failed", {- module: "rate-limit-experiment",- orgId: caller.orgId,- error,- });- return undefined;- } finally {- pendingAssignments.delete(caller.orgId);- }- })();- pendingAssignments.set(caller.orgId, promise);- return promise;-}--function refreshAssignment(caller: ExperimentCaller): void {- void fetchAssignment(caller).then(fresh => {- if (fresh) {- setBounded(localAssignments, caller.orgId, {- assignment: fresh,- until: Date.now() + LOCAL_TTL_MS,- });- }- });-}--async function getAssignment(- caller: ExperimentCaller,-): Promise<Assignment | undefined> {- const local = localAssignments.get(caller.orgId);- if (local && local.until > Date.now()) return local.assignment;-- const stored = await readAssignment(caller.orgId);- let assignment: Assignment | undefined;- if (stored === STORE_UNAVAILABLE) {- // The org may well have a stored assignment, so keep this process's last- // answer rather than wait on PostHog, and ask it in the background.- assignment = local?.assignment;- refreshAssignment(caller);- } else if (!stored) {- // No stored answer yet, e.g. the org's first request: the only case that- // waits on PostHog.- assignment = await fetchAssignment(caller);- } else {- assignment = stored;- if (Date.now() - stored.checkedAt > ASSIGNMENT_FRESH_MS) {- refreshAssignment(caller);- }- }- setBounded(localAssignments, caller.orgId, {- assignment,- until: Date.now() + LOCAL_TTL_MS,- });- return assignment;-}--/**- * Send the experiment exposure, `$feature_flag_called` with the variant, once- * per (org, variant) while its marker lasts (EXPOSURE_TTL_SECONDS), the same- * event the web app's server-resolved experiments send. An org moved to the- * other variant is reported again, and PostHog excludes it as a multiple- * exposure. Fire-and-forget; never throws.- */-function trackExposure(- caller: ExperimentCaller,- variant: RateLimitExperimentVariant,-): void {- const marker = `${caller.orgId}:${variant}`;- if ((skipExposureUntil.get(marker) ?? 0) > Date.now()) return;- if (pendingExposures.has(marker)) return;- pendingExposures.add(marker);-- void (async () => {- try {- const key = `firecrawl:rate_limit_experiment_exposure:${marker}`;- const isFirst = await redisEvictConnection.set(- key,- "1",- "EX",- EXPOSURE_PENDING_TTL_SECONDS,- "NX",- );- if (isFirst !== "OK") {- // Another process holds the marker. Check again once a pending marker- // would have expired, in case that process's send failed.- setBounded(- skipExposureUntil,- marker,- Date.now() + EXPOSURE_PENDING_TTL_SECONDS * 1000,- );- return;- }- const distinctId = await resolveDistinctId(- caller.teamId,- caller.apiKeyId,- );- const sent = await capturePostHog("$feature_flag_called", distinctId, {- $feature_flag: RATE_LIMIT_EXPERIMENT_FLAG,- $feature_flag_response: variant,- team_id: caller.teamId,- rate_limit_multiplier: caller.rateLimitMultiplier,- is_paid_plan: caller.isPaidPlan,- $groups: { company: caller.orgId, team: caller.teamId },- });- if (sent) {- setBounded(skipExposureUntil, marker, Number.POSITIVE_INFINITY);- // Best effort: if this fails, the pending marker expires and another- // process may resend an exposure PostHog already counted once.- await redisEvictConnection- .expire(key, EXPOSURE_TTL_SECONDS)- .catch(error => {- _logger.debug(- "Rate-limit experiment exposure marker not extended",- {- module: "rate-limit-experiment",- orgId: caller.orgId,- error,- },- );- });- } else {- // A rejected capture must not use up the exposure, but is not retried- // on every request while PostHog keeps rejecting it.- setBounded(- skipExposureUntil,- marker,- Date.now() + EXPOSURE_PENDING_TTL_SECONDS * 1000,- );- await redisEvictConnection.del(key);- }- } catch (error) {- _logger.debug("Rate-limit experiment exposure failed", {- module: "rate-limit-experiment",- orgId: caller.orgId,- error,- });- } finally {- pendingExposures.delete(marker);- }- })();-}--/**- * The org's rate-limit experiment variant, reported to PostHog as an exposure- * the first time it is seen. Undefined when the org is not enrolled: PostHog- * capture is off, the flag is off or unreachable, or the variant is unknown;- * the caller then keeps the current limits. Call it only for a request whose- * limit the variant changes (see rateLimitExperimentApplies), once the request- * has passed the auth checks.- */-export async function getRateLimitExperimentVariant(- caller: ExperimentCaller,-): Promise<RateLimitExperimentVariant | undefined> {- if (!isPostHogCaptureEnabled() || !caller.orgId) return undefined;- const variant = (await getAssignment(caller))?.variant ?? undefined;- if (variant !== undefined) trackExposure(caller, variant);- return variant;-}diff --git a/apps/api/src/services/rate-limiter.test.ts b/apps/api/src/services/rate-limiter.test.tsindex 418d1735e..53331fb1e 100644--- a/apps/api/src/services/rate-limiter.test.ts+++ b/apps/api/src/services/rate-limiter.test.ts@@ -15,7 +15,6 @@ import {getAutumnRateLimiter,getHostedMcpConcurrencyLimit,getHostedMcpRateLimiter,- rateLimitExperimentApplies,} from "./rate-limiter";// import {@@ -422,75 +421,6 @@ describe("getAutumnRateLimiter", () => {).toBe(10000);});- describe("rate-limit experiment", () => {- const limit = (- mode: RateLimiterMode,- multiplier: number,- variant?: "control" | "treatment",- ) => getAutumnRateLimiter(mode, multiplier, undefined, variant).points;-- it("raises scrape, search and exchange to the plan's minimum for treatment", () => {- for (const mode of [- RateLimiterMode.Scrape,- RateLimiterMode.Search,- RateLimiterMode.Exchange,- ]) {- expect(limit(mode, 1, "treatment")).toBe(50);- expect(limit(mode, 10, "treatment")).toBe(500);- expect(limit(mode, 50, "treatment")).toBe(1000);- expect(limit(mode, 500, "treatment")).toBe(5000);- }- });-- it("keeps base × multiplier for control and without a variant", () => {- expect(limit(RateLimiterMode.Scrape, 1, "control")).toBe(10);- expect(limit(RateLimiterMode.Search, 10, "control")).toBe(100);- expect(limit(RateLimiterMode.Exchange, 50)).toBe(500);- });-- it("rounds an off-tier multiplier down to the tier below it", () => {- expect(limit(RateLimiterMode.Scrape, 5, "treatment")).toBe(50);- expect(limit(RateLimiterMode.Scrape, 6, "treatment")).toBe(60);- expect(limit(RateLimiterMode.Scrape, 20, "treatment")).toBe(500);- expect(limit(RateLimiterMode.Scrape, 200, "treatment")).toBe(2000);- });-- it("leaves other modes and overrides alone for treatment", () => {- expect(limit(RateLimiterMode.Map, 1, "treatment")).toBe(10);- expect(limit(RateLimiterMode.Crawl, 1, "treatment")).toBe(2);- expect(- getAutumnRateLimiter(- RateLimiterMode.Scrape,- 1,- flagsWith({ [RateLimiterMode.Scrape]: 5 }),- "treatment",- ).points,- ).toBe(5);- });-- it("applies only where the treatment minimum changes the limit", () => {- expect(rateLimitExperimentApplies(RateLimiterMode.Scrape, 1)).toBe(true);- expect(rateLimitExperimentApplies(RateLimiterMode.Search, 10)).toBe(true);- expect(rateLimitExperimentApplies(RateLimiterMode.Exchange, 50)).toBe(- true,- );- expect(rateLimitExperimentApplies(RateLimiterMode.Scrape, 20)).toBe(true);- // 50/min in both variants, or above every minimum.- expect(rateLimitExperimentApplies(RateLimiterMode.Scrape, 5)).toBe(false);- expect(rateLimitExperimentApplies(RateLimiterMode.Scrape, 500)).toBe(- false,- );- expect(rateLimitExperimentApplies(RateLimiterMode.Map, 1)).toBe(false);- expect(- rateLimitExperimentApplies(- RateLimiterMode.Scrape,- 1,- flagsWith({ [RateLimiterMode.Scrape]: 5 }),- ),- ).toBe(false);- });- });-describe("hosted MCP minimum", () => {it("raises free and hobby to 600 and standard to 2,000 in a bucket of its own", () => {for (const mode of [RateLimiterMode.Scrape, RateLimiterMode.Search]) {diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.tsindex 70762952d..4f8bea2da 100644--- a/apps/api/src/services/rate-limiter.ts+++ b/apps/api/src/services/rate-limiter.ts@@ -46,10 +46,9 @@ const fallbackRateLimits: Record<RateLimiterMode, number> = {/*** Per-minute base rate limits, i.e. the ×1 values. The effective limit is- * `base × multiplier`, raised to the minimum (see treatmentMinimum) for orgs in- * the "treatment" variant, and the multiplier is read from Autumn's- * `rate_limits` feature. Modes absent here are not multiplier-scaled and use- * the fallback table.+ * `base × multiplier`, where the multiplier is read from Autumn's `rate_limits`+ * feature. Modes absent here are not multiplier-scaled and use the fallback+ * table.** Endpoint → mode mapping: agent + extract share `Extract`; interact is* `Browser`; interactExecute is `BrowserExecute`; agentStatus is@@ -103,20 +102,17 @@ export function getRateLimitOverride(/*** Builds the per-minute rate limiter for an authenticated team from its Autumn* rate-limit multiplier: the effective limit is `base × multiplier` for- * multiplier-scaled modes (default ×1), raised to the minimum for the- * "treatment" variant of the rate-limit experiment (see- * services/rate-limit-experiment.ts). Modes without a base fall back to the+ * multiplier-scaled modes (default ×1). Modes without a base fall back to the* static table.** An org-level override for the mode replaces that whole computation, so the- * multiplier, the base table and the minimum are all discarded. Modes without- * an override keep the normal result.+ * multiplier and the base table are both discarded. Modes without an override+ * keep the normal result.*/export function getAutumnRateLimiter(mode: RateLimiterMode,multiplier: number = 1,flags?: TeamFlags,- variant?: RateLimitExperimentVariant,): RateLimiterRedis {const override = getRateLimitOverride(mode, flags?.rateLimitOverrides);if (override !== undefined) {@@ -127,13 +123,7 @@ export function getAutumnRateLimiter(let rateLimit: number;if (base !== undefined) {const safeMultiplier = multiplier > 0 ? multiplier : 1;- rateLimit =- variant === "treatment"- ? Math.max(- base * safeMultiplier,- treatmentMinimum(mode, safeMultiplier),- )- : base * safeMultiplier;+ rateLimit = base * safeMultiplier;} else {rateLimit = fallbackRateLimits?.[mode] ?? 500;}@@ -251,63 +241,6 @@ export function getHostedMcpConcurrencyLimit(*/export const HOBBY_RATE_LIMIT_MULTIPLIER = 10;-/**- * Modes that get the treatment minimum. Agent traffic is bursty, so the scrape- * and search limits refused valid calls. Exchange keeps parity with Scrape, as- * in the base table.- */-const TREATMENT_MINIMUM_MODES: ReadonlySet<RateLimiterMode> = new Set([- RateLimiterMode.Scrape,- RateLimiterMode.Exchange,- RateLimiterMode.Search,-]);--/**- * Per-minute minimums for orgs in the "treatment" variant, keyed by the lowest- * Autumn rate-limit multiplier each applies to. An org takes the entry for the- * highest multiplier it meets, so off-tier multipliers round down, and the- * limit is `max(base × multiplier, minimum)`. Raising a floor instead of the- * base leaves every plan whose `base × multiplier` is already higher, growth- * and above, unchanged.- */-const TREATMENT_MINIMUM_TIERS: { minMultiplier: number; limit: number }[] = [- { minMultiplier: 1, limit: 50 }, // free- { minMultiplier: HOBBY_RATE_LIMIT_MULTIPLIER, limit: 500 }, // hobby- { minMultiplier: 50, limit: 1000 }, // standard-];--/** The treatment minimum for a mode and multiplier, or 0 when there is none. */-function treatmentMinimum(mode: RateLimiterMode, multiplier: number): number {- if (!TREATMENT_MINIMUM_MODES.has(mode)) return 0;- let minimum = 0;- for (const tier of TREATMENT_MINIMUM_TIERS) {- if (multiplier >= tier.minMultiplier) minimum = tier.limit;- }- return minimum;-}--export type RateLimitExperimentVariant = "control" | "treatment";--/**- * Whether the rate-limit experiment decides this limit: true only when the- * "treatment" minimum is above `base × multiplier`. False under an org- * override, for a mode without a minimum, and for a plan that already meets- * it, where both variants get the same limit.- */-export function rateLimitExperimentApplies(- mode: RateLimiterMode,- multiplier: number,- flags?: TeamFlags,-): boolean {- if (getRateLimitOverride(mode, flags?.rateLimitOverrides) !== undefined) {- return false;- }- const base = BASE_RATE_LIMITS[mode];- if (base === undefined) return false;- const safeMultiplier = multiplier > 0 ? multiplier : 1;- return treatmentMinimum(mode, safeMultiplier) > base * safeMultiplier;-}-/*** Plan-priority tiers keyed by the minimum Autumn rate-limit multiplier that* qualifies. Values mirror the tuned production `plan_priority` for each plan.