Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(browser): floor trusted agent browser concurrency at the hobby limit (#4811)
Rakshith Ramprakash5507d6059e42
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 12d9fd271..394be5b68 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -238,6 +238,18 @@ jobs:run: pnpm exec vitest run src/controllers/v2/__tests__/browser-delete.test.ts --reporter=defaultworking-directory: apps/api+ - name: Run agent interop auth and concurrency regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/lib/agent-interop.test.ts+ src/controllers/__tests__/auth.test.ts+ src/__tests__/routes/exchange-agent-managed-key.routes.test.ts+ src/controllers/v2/__tests__/browser-agent-concurrency.test.ts+ src/__tests__/routes/interact-agent-concurrency.routes.test.ts+ --reporter=default+ working-directory: apps/api+- name: Install test site dependenciesrun: pnpm install --frozen-lockfileworking-directory: apps/test-sitediff --git a/apps/api/src/__tests__/routes/interact-agent-concurrency.routes.test.ts b/apps/api/src/__tests__/routes/interact-agent-concurrency.routes.test.tsnew file mode 100644index 000000000..6812e6744--- /dev/null+++ b/apps/api/src/__tests__/routes/interact-agent-concurrency.routes.test.ts@@ -0,0 +1,327 @@+import express from "express";+import request from "supertest";++// The real authMiddleware, auth.ts and scrapeInteractController (which+// re-parses req.body and drops `__agentInterop`); only I/O is mocked.+const mocks = vi.hoisted(() => ({+ authChunk: vi.fn(),+ reserveExternalSlot: vi.fn(),+}));++vi.mock("../../config", async importOriginal => {+ const actual = await importOriginal<typeof import("../../config")>();+ return {+ ...actual,+ config: {+ ...actual.config,+ USE_DB_AUTHENTICATION: true,+ HANGAR_URL: "http://hangar.test",+ AGENT_INTEROP_SECRET: "agent-secret",+ },+ };+});+vi.mock("../../db/connection", () => ({ db: {}, dbRr: {}, dbIndex: {} }));+vi.mock("../../db/rpc", () => ({+ authCreditUsageChunk: mocks.authChunk,+ authCreditUsageChunkFromTeam: vi.fn(),+ getAgentFreeRequestsLeft: vi.fn(),+}));+vi.mock("../../services/redis", () => ({+ getValue: vi.fn().mockResolvedValue(null),+ setValue: vi.fn(),+ deleteKey: vi.fn(),+}));+vi.mock("../../services/redlock", () => ({+ redlock: { using: vi.fn(async (_k, _t, _o, fn) => fn({ aborted: false })) },+}));+vi.mock("ioredis", () => ({ default: class {} }));+vi.mock("../../services/rate-limiter", async importOriginal => ({+ ...(await importOriginal<typeof import("../../services/rate-limiter")>()),+ getRateLimiter: vi.fn(() => ({ consume: vi.fn() })),+ getAutumnRateLimiter: vi.fn(() => ({ consume: vi.fn() })),+ redisRateLimitClient: {+ set: vi.fn().mockResolvedValue("OK"),+ get: vi.fn().mockResolvedValue(null),+ del: vi.fn().mockResolvedValue(1),+ eval: vi.fn().mockResolvedValue(1),+ },+}));+vi.mock("../../services/autumn/autumn.service", async importOriginal => {+ const actual =+ await importOriginal<+ typeof import("../../services/autumn/autumn.service")+ >();+ return {+ ...actual,+ autumnService: {+ getRateLimitMultiplier: vi.fn().mockResolvedValue(1),+ getKnownRateLimitMultiplier: vi.fn().mockResolvedValue(1),+ // Free plan: Autumn grants CONCURRENCY 2.+ getConcurrencyLimit: vi.fn().mockResolvedValue(2),+ checkCredits: vi.fn().mockResolvedValue(null),+ },+ };+});+vi.mock("../../services/agent-sponsor", () => ({+ getAgentSponsorStatus: vi.fn().mockResolvedValue(null),+}));+vi.mock("../../services/queue-service", () => ({+ getRedisConnection: vi.fn(() => ({ sadd: vi.fn() })),+ getBillingQueue: vi.fn(() => ({ add: vi.fn() })),+}));+vi.mock("../../services/logging/log_job", () => ({+ logRequest: vi.fn(),+ logProviderScrape: vi.fn(),+}));+vi.mock("../../services/worker/nuq-router", () => ({+ scrapeQueue: { getJob: vi.fn(async () => null) },+ getCombinedTeamActiveCount: vi.fn(async () => 0),+ reserveExternalSlot: mocks.reserveExternalSlot,+ mirrorExternalSlotRelease: vi.fn(async () => {}),+}));+vi.mock("../../lib/operational-job-access", () => ({+ getScrapeJobAccess: vi.fn(async () => ({+ teamId: "team-free",+ expiresAtMs: Date.now() + 60_000,+ })),+}));+vi.mock("../../lib/job-state-store", () => ({+ readScrapeJobState: vi.fn(async () => null),+}));+vi.mock("../../lib/job-store-fallback", () => ({+ recordJobStorePostgresFallback: vi.fn(),+}));+vi.mock("../../lib/supabase-jobs", () => ({+ supabaseGetScrapeByIdDirect: vi.fn(async () => ({+ id: "scrape-1",+ team_id: "team-free",+ url: "https://example.com",+ options: {},+ })),+}));+vi.mock("../../lib/browser-sessions", () => ({+ insertBrowserSession: vi.fn(async (row: unknown) => row),+ completeBrowserSessionSettlement: vi.fn(async () => {}),+ getBrowserSession: vi.fn(async () => null),+ getBrowserSessionFromScrape: vi.fn(async () => null),+ listUnsettledHangarSessions: vi.fn(async () => []),+ updateBrowserSessionActivity: vi.fn(async () => {}),+ updateBrowserSessionScrapeId: vi.fn(async () => {}),+ settleBrowserSessionOnce: vi.fn(),+ markBrowserSessionUsedPrompt: vi.fn(async () => {}),+ didBrowserSessionUsePrompt: vi.fn(),+ upsertBrowserProfile: vi.fn(),+}));+vi.mock("../../lib/hangar", async importOriginal => ({+ ...(await importOriginal<typeof import("../../lib/hangar")>()),+ createHangarBrowser: vi.fn(async () => ({+ id: "br_1",+ cdp_url: "wss://cdp.test",+ view_url: null,+ control_url: null,+ playlist_url: null,+ max_expires_at: null,+ })),+ executeHangarBrowser: vi.fn(async () => ({+ stdout: "",+ result: "",+ stderr: "",+ exitCode: 0,+ killed: false,+ })),+ stopHangarBrowser: vi.fn(async () => {}),+}));+vi.mock("../../lib/scrape-interact/browser-agent", () => ({+ selectBrowserAgentTab: vi.fn(async () => {}),+ executePromptViaBrowserAgent: vi.fn(),+ executeCodeViaBrowserSession: vi.fn(async () => ({+ stdout: "",+ result: "",+ stderr: "",+ exitCode: 0,+ killed: false,+ })),+}));+vi.mock("../../lib/browser-session-activity", () => ({+ enqueueBrowserSessionActivity: vi.fn(),+}));+vi.mock("../../services/billing/credit_billing", () => ({+ billTeam: vi.fn(async () => {}),+}));++import { authMiddleware } from "../../routes/shared";+import { RateLimiterMode } from "../../types";+import { scrapeInteractController } from "../../controllers/v2/scrape-browser";+import { HOBBY_CONCURRENCY_LIMIT } from "../../lib/concurrency-limit";+import { getBrowserSessionFromScrape } from "../../lib/browser-sessions";+import { executeCodeViaBrowserSession } from "../../lib/scrape-interact/browser-agent";++const API_KEY = "33333333-3333-4333-8333-333333333333";+const SECRET = "agent-secret";+const FREE_LIMIT = 2;++const app = express();+app.use(express.json());+app.post(+ "/v2/scrape/:jobId/interact",+ authMiddleware(RateLimiterMode.BrowserExecute),+ (req, res, next) => {+ scrapeInteractController(req as any, res as any).catch(next);+ },+);++const WRONG = "not-the-secret";++// [body secret, header secret] for each case; undefined means not sent.+const INTEROP = {+ none: [undefined, undefined],+ body: [SECRET, undefined],+ header: [undefined, SECRET],+ both: [SECRET, SECRET],+ "wrong-body": [WRONG, undefined],+ "wrong-header": [undefined, WRONG],+ "valid-header-wrong-body": [WRONG, SECRET],+ "wrong-header-valid-body": [SECRET, WRONG],+} as const;++type Interop = keyof typeof INTEROP | "forged";++function interact(interop: Interop) {+ const [bodySecret, headerSecret] =+ interop === "forged" ? [undefined, undefined] : INTEROP[interop];+ let body: Record<string, unknown> = { code: "console.log(1)" };+ if (bodySecret !== undefined) {+ body.__agentInterop = {+ auth: bodySecret,+ requestId: "11111111-1111-4111-8111-111111111111",+ shouldBill: true,+ };+ }+ if (interop === "forged") {+ body = {+ ...body,+ trustedAgentInterop: true,+ auth: { team_id: "team-free", trustedAgentInterop: true },+ };+ }+ let req = request(app)+ .post("/v2/scrape/scrape-1/interact")+ .set("authorization", `Bearer ${API_KEY}`);+ if (headerSecret !== undefined)+ req = req.set("x-firecrawl-agent-interop", headerSecret);+ return req.send(body);+}++/** The limit the interact path passed to the browser slot reservation. */+async function reservedLimit(interop: Interop): Promise<number> {+ await interact(interop);+ expect(mocks.reserveExternalSlot).toHaveBeenCalledTimes(1);+ return mocks.reserveExternalSlot.mock.calls[0][3];+}++describe("interact browser concurrency for trusted agent requests", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mocks.reserveExternalSlot.mockResolvedValue(true);+ vi.mocked(getBrowserSessionFromScrape).mockResolvedValue(null);+ mocks.authChunk.mockResolvedValue([+ {+ api_key: API_KEY,+ api_key_id: 9,+ api_key_id_text: "9",+ team_id: "team-free",+ org_id: "org-free",+ is_banned: false,+ flags: null,+ credential_purpose: "general",+ },+ ]);+ });++ it("floors a free team at hobby when the secret is only in the body", async () => {+ expect(await reservedLimit("body")).toBe(HOBBY_CONCURRENCY_LIMIT);+ });++ it("floors a free team at hobby when the secret is only in the header", async () => {+ expect(await reservedLimit("header")).toBe(HOBBY_CONCURRENCY_LIMIT);+ });++ it("floors a free team at hobby when both secrets are valid", async () => {+ expect(await reservedLimit("both")).toBe(HOBBY_CONCURRENCY_LIMIT);+ });++ it.each([+ "wrong-body",+ "wrong-header",+ "valid-header-wrong-body",+ "wrong-header-valid-body",+ ] as const)("rejects %s with the same 403 as /v2/browser", async interop => {+ const res = await interact(interop);+ expect(res.status).toBe(403);+ expect(res.body).toEqual({+ success: false,+ error: "Invalid agent interop.",+ });+ expect(mocks.reserveExternalSlot).not.toHaveBeenCalled();+ });++ it("serves a request without interop at the free limit", async () => {+ expect(await reservedLimit("none")).toBe(FREE_LIMIT);+ });++ it("ignores a forged trustedAgentInterop field in the body", async () => {+ expect(await reservedLimit("forged")).toBe(FREE_LIMIT);+ });+});++describe("interact on an existing browser session", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mocks.authChunk.mockResolvedValue([+ {+ api_key: API_KEY,+ api_key_id: 9,+ api_key_id_text: "9",+ team_id: "team-free",+ org_id: "org-free",+ is_banned: false,+ flags: null,+ credential_purpose: "general",+ },+ ]);+ vi.mocked(getBrowserSessionFromScrape).mockResolvedValue({+ id: "session-1",+ team_id: "team-free",+ browser_id: "br_1",+ status: "active",+ cdp_url: "wss://cdp.test",+ cdp_path: "",+ cdp_interactive_path: "",+ } as any);+ });++ it.each([+ "wrong-body",+ "wrong-header",+ "valid-header-wrong-body",+ "wrong-header-valid-body",+ ] as const)("rejects %s with 403 and executes nothing", async interop => {+ const res = await interact(interop);+ expect(res.status).toBe(403);+ expect(res.body).toEqual({+ success: false,+ error: "Invalid agent interop.",+ });+ expect(executeCodeViaBrowserSession).not.toHaveBeenCalled();+ });++ it.each(["body", "header", "both", "none"] as const)(+ "executes on the reused session with %s interop",+ async interop => {+ const res = await interact(interop);+ expect(res.status).toBe(200);+ expect(executeCodeViaBrowserSession).toHaveBeenCalledTimes(1);+ expect(mocks.reserveExternalSlot).not.toHaveBeenCalled();+ },+ );+});diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex a0c8c3a3b..485726514 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -11,10 +11,7 @@ import {getRateLimitOverride,HOBBY_RATE_LIMIT_MULTIPLIER,} from "../services/rate-limiter";-import {- AGENT_INTEROP_HEADER,- isAgentInteropSecretValid,-} from "../lib/agent-interop";+import { isTrustedAgentInteropRequest } from "../lib/agent-interop";import {KEYLESS_FREE_TIER_LIMIT_MESSAGE,consumeKeylessRequest,@@ -698,19 +695,6 @@ type AuthenticateOptions = {allowAgentManagedKey?: boolean;};-/**- * Whether the request carries a valid agent-interop secret, i.e. comes from- * the trusted internal agent service. Read from the raw body (auth runs before- * the controller's zod parse) or, for bodiless calls, from AGENT_INTEROP_HEADER.- * Presence of the block or header alone is never trusted; only the secret.- */-function isTrustedAgentInteropRequest(req): boolean {- return (- isAgentInteropSecretValid(req.body?.__agentInterop?.auth) ||- isAgentInteropSecretValid(req.headers?.[AGENT_INTEROP_HEADER])- );-}-async function supaAuthenticateUser(req,res,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 9d6386358..fd945c804 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1,3 +1,4 @@+import type { AgentInteropStatus } from "../../lib/agent-interop";import { Request } from "express";import { config } from "../../config";import { z } from "zod";@@ -1309,6 +1310,9 @@ export type CrawlErrorsResponse =type AuthObject = {team_id: string;org_id?: string | null;+ // Set only by authMiddleware from the raw request; controllers may re-parse+ // the body and drop `__agentInterop`, so read this instead.+ agentInterop?: AgentInteropStatus;};type Account = {diff --git a/apps/api/src/controllers/v2/__tests__/browser-agent-concurrency.test.ts b/apps/api/src/controllers/v2/__tests__/browser-agent-concurrency.test.tsnew file mode 100644index 000000000..c3eba8b2d--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/browser-agent-concurrency.test.ts@@ -0,0 +1,258 @@+import type { Response } from "express";+import { v4 as uuidv4 } from "uuid";++const SECRET = "agent-secret";++const mocks = vi.hoisted(() => ({+ teamConcurrency: null as number | null,+ held: new Set<string>(),+ reserveExternalSlot: vi.fn(),+}));++vi.mock("../../../config", async importOriginal => {+ const actual = await importOriginal<typeof import("../../../config")>();+ return {+ ...actual,+ config: {+ ...actual.config,+ HANGAR_URL: "http://hangar.test",+ AGENT_INTEROP_SECRET: "agent-secret",+ },+ };+});++vi.mock("../../../lib/logger", () => {+ const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() };+ const child = vi.fn(() => ({ ...log, child }));+ return { logger: { ...log, child } };+});++vi.mock("../../../services/autumn/autumn.service", () => ({+ autumnService: {+ getConcurrencyLimit: vi.fn(async () => mocks.teamConcurrency),+ checkCredits: vi.fn(async () => null),+ },+}));++// Same admission rule as the real slot ledger: admit while under the limit.+vi.mock("../../../services/worker/nuq-router", () => ({+ reserveExternalSlot: mocks.reserveExternalSlot,+ mirrorExternalSlotRelease: vi.fn(async (_team: string, id: string) => {+ mocks.held.delete(id);+ }),+ getCombinedTeamActiveCount: vi.fn(async () => mocks.held.size),+}));++vi.mock("../../../lib/hangar", async importOriginal => {+ const actual = await importOriginal<typeof import("../../../lib/hangar")>();+ return {+ ...actual,+ createHangarBrowser: vi.fn(async (key: string) => ({+ id: `browser-${key}`,+ cdp_url: "wss://cdp.test",+ view_url: null,+ control_url: null,+ playlist_url: null,+ max_expires_at: null,+ })),+ stopHangarBrowser: vi.fn(async () => {}),+ };+});++vi.mock("../../../lib/browser-sessions", () => ({+ insertBrowserSession: vi.fn(async (row: { id: string }) => ({+ ...row,+ cdp_url: "wss://cdp.test",+ cdp_path: "",+ cdp_interactive_path: "",+ })),+ completeBrowserSessionSettlement: vi.fn(),+ getBrowserSession: vi.fn(),+ getBrowserSessionFromScrape: vi.fn(),+ listBrowserSessions: vi.fn(),+ updateBrowserSessionActivity: vi.fn(),+ updateBrowserSessionScrapeId: vi.fn(),+ settleBrowserSessionOnce: vi.fn(),+ withLockedBrowserSession: vi.fn(),+ listUnsettledHangarSessions: vi.fn(),+ didBrowserSessionUsePrompt: vi.fn(),+ markBrowserSessionUsedPrompt: vi.fn(),+ upsertBrowserProfile: vi.fn(),+ deleteBrowserProfile: vi.fn(),+}));++vi.mock("../../../services/logging/log_job", () => ({+ logRequest: vi.fn(async () => {}),+}));++import { browserCreateController } from "../browser";+import { HOBBY_CONCURRENCY_LIMIT } from "../../../lib/concurrency-limit";+import { agentInteropStatus } from "../../../lib/agent-interop";++const TEAM_ID = "11111111-1111-1111-1111-111111111111";+const FREE_LIMIT = 2;++function makeRes() {+ return {+ statusCode: 200,+ body: undefined as any,+ status(code: number) {+ this.statusCode = code;+ return this;+ },+ json(body: unknown) {+ this.body = body;+ return this;+ },+ };+}++const WRONG = "not-the-secret";++// [body secret, header secret] for each case; undefined means not sent.+const INTEROP = {+ none: [undefined, undefined],+ body: [SECRET, undefined],+ header: [undefined, SECRET],+ both: [SECRET, SECRET],+ "wrong-body": [WRONG, undefined],+ "wrong-header": [undefined, WRONG],+ "valid-header-wrong-body": [WRONG, SECRET],+ "wrong-header-valid-body": [SECRET, WRONG],+} as const;++type Interop = keyof typeof INTEROP;++function makeReq(interop: Interop) {+ const [bodySecret, headerSecret] = INTEROP[interop];+ const body: Record<string, unknown> = {};+ const headers: Record<string, string> = {};+ if (bodySecret !== undefined) {+ body.__agentInterop = {+ auth: bodySecret,+ requestId: uuidv4(),+ shouldBill: false,+ };+ }+ if (headerSecret !== undefined)+ headers["x-firecrawl-agent-interop"] = headerSecret;+ return {+ body,+ headers,+ path: "/v2/browser",+ // As authMiddleware sets it, from the raw request.+ auth: {+ team_id: TEAM_ID,+ agentInterop: agentInteropStatus({ body, headers }),+ },+ acuc: { org_id: "org-1", api_key_id: null, flags: null },+ };+}++/** Creates browsers until one is refused; returns how many were admitted. */+async function admittedUntilRefused(interop: Interop, max = 100) {+ for (let i = 0; i < max; i++) {+ const res = makeRes();+ await browserCreateController(makeReq(interop) as any, res as any);+ if (res.statusCode !== 200) {+ expect(res.statusCode).toBe(429);+ return { admitted: i, error: res.body.error as string };+ }+ }+ throw new Error("never refused");+}++describe("browser create concurrency for trusted agent requests", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mocks.held.clear();+ mocks.teamConcurrency = null;+ mocks.reserveExternalSlot.mockImplementation(+ async (_team: string, id: string, _ttl: number, limit: number) => {+ if (mocks.held.size >= limit) return false;+ mocks.held.add(id);+ return true;+ },+ );+ });++ it("lifts a free team's trusted agent (body secret) to the hobby limit", async () => {+ const { admitted, error } = await admittedUntilRefused("body");+ expect(admitted).toBe(HOBBY_CONCURRENCY_LIMIT);+ expect(error).toContain(`(${HOBBY_CONCURRENCY_LIMIT})`);+ });++ it("lifts a free team's trusted agent (header secret) to the hobby limit", async () => {+ const { admitted } = await admittedUntilRefused("header");+ expect(admitted).toBe(HOBBY_CONCURRENCY_LIMIT);+ });++ it("keeps the same free team at the free limit without interop", async () => {+ const { admitted, error } = await admittedUntilRefused("none");+ expect(admitted).toBe(FREE_LIMIT);+ expect(error).toContain(`(${FREE_LIMIT})`);+ });++ it("rejects a wrong header secret before any slot is reserved", async () => {+ const res = makeRes();+ await browserCreateController(makeReq("wrong-header") as any, res as any);+ expect(res.statusCode).toBe(403);+ expect(res.body.error).toBe("Invalid agent interop.");+ expect(mocks.reserveExternalSlot).not.toHaveBeenCalled();+ });++ it("rejects a wrong body secret before any slot is reserved", async () => {+ const res = makeRes();+ await browserCreateController(makeReq("wrong-body") as any, res as any);+ expect(res.statusCode).toBe(403);+ expect(mocks.reserveExternalSlot).not.toHaveBeenCalled();+ });++ it("counts slots already held by non-agent work toward the floor", async () => {+ mocks.held.add("scrape-1");+ mocks.held.add("scrape-2");+ const { admitted } = await admittedUntilRefused("body");+ expect(admitted).toBe(HOBBY_CONCURRENCY_LIMIT - 2);+ });++ it("leaves a plan above hobby unchanged for trusted agent requests", async () => {+ mocks.teamConcurrency = 50;+ const { admitted } = await admittedUntilRefused("body");+ expect(admitted).toBe(50);+ });++ it("keeps a team-specific Autumn grant above hobby as-is", async () => {+ mocks.teamConcurrency = HOBBY_CONCURRENCY_LIMIT + 3;+ expect((await admittedUntilRefused("body")).admitted).toBe(+ HOBBY_CONCURRENCY_LIMIT + 3,+ );+ mocks.held.clear();+ expect((await admittedUntilRefused("none")).admitted).toBe(+ HOBBY_CONCURRENCY_LIMIT + 3,+ );+ });++ it("passes the floored limit only to the browser reservation", async () => {+ await browserCreateController(makeReq("body") as any, makeRes() as any);+ await browserCreateController(makeReq("none") as any, makeRes() as any);+ const limits = mocks.reserveExternalSlot.mock.calls.map(c => c[3]);+ expect(limits).toEqual([HOBBY_CONCURRENCY_LIMIT, FREE_LIMIT]);+ });++ it.each(["valid-header-wrong-body", "wrong-header-valid-body"] as const)(+ "rejects %s with 403 before any slot is reserved",+ async interop => {+ const res = makeRes();+ await browserCreateController(makeReq(interop) as any, res as any);+ expect(res.statusCode).toBe(403);+ expect(res.body.error).toBe("Invalid agent interop.");+ expect(mocks.reserveExternalSlot).not.toHaveBeenCalled();+ },+ );++ it("floors a free team at hobby when both secrets are valid", async () => {+ expect((await admittedUntilRefused("both")).admitted).toBe(+ HOBBY_CONCURRENCY_LIMIT,+ );+ });+});diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 52157ad28..ffd67d8fb 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -18,6 +18,7 @@ import {reserveBrowserPromptCredits,stopBrowserSession,browserSessionLinks,+ invalidAgentInteropError,} from "../../lib/browser-lifecycle";import { browserCreateRequestSchema, browserError } from "./browser";import {@@ -107,6 +108,14 @@ export async function scrapeInteractController(>,res: Response<BrowserExecuteResponse>,) {+ // Before the reuse-or-create branch, so a reused session is guarded too.+ const invalidInterop = invalidAgentInteropError(req);+ if (invalidInterop) {+ return res+ .status(invalidInterop.status)+ .json({ success: false, error: invalidInterop.message });+ }+req.body = browserExecuteRequestSchema.parse(req.body);if (getSafeMode(req.acuc?.flags)) {diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 18bf6ea39..68f0c7e0a 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1,3 +1,4 @@+import type { AgentInteropStatus } from "../../lib/agent-interop";import { Request, Response } from "express";import { hasCategory } from "../../lib/search-query-builder";import { config } from "../../config";@@ -2019,6 +2020,9 @@ export type CrawlErrorsResponse =type AuthObject = {team_id: string;org_id?: string | null;+ // Set only by authMiddleware from the raw request; controllers may re-parse+ // the body and drop `__agentInterop`, so read this instead.+ agentInterop?: AgentInteropStatus;};type Account = {diff --git a/apps/api/src/lib/agent-interop.test.ts b/apps/api/src/lib/agent-interop.test.tsnew file mode 100644index 000000000..47abfd4be--- /dev/null+++ b/apps/api/src/lib/agent-interop.test.ts@@ -0,0 +1,59 @@+vi.mock("../config", () => ({+ config: { AGENT_INTEROP_SECRET: "agent-secret" },+}));++import {+ AGENT_INTEROP_HEADER,+ agentInteropStatus,+ isTrustedAgentInteropRequest,+} from "./agent-interop";++const GOOD = "agent-secret";+const BAD = "not-the-secret";++type Place = typeof GOOD | typeof BAD | undefined;++function req(body: Place, header: Place) {+ return {+ body:+ body === undefined+ ? {}+ : { __agentInterop: { auth: body, requestId: "r", shouldBill: true } },+ headers: header === undefined ? {} : { [AGENT_INTEROP_HEADER]: header },+ };+}++describe("agentInteropStatus", () => {+ it.each([+ ["nothing sent", "none", undefined, undefined],+ ["valid body only", "trusted", GOOD, undefined],+ ["valid header only", "trusted", undefined, GOOD],+ ["both valid", "trusted", GOOD, GOOD],+ ["wrong body only", "invalid", BAD, undefined],+ ["wrong header only", "invalid", undefined, BAD],+ ["valid header, wrong body", "invalid", BAD, GOOD],+ ["wrong header, valid body", "invalid", GOOD, BAD],+ ["both wrong", "invalid", BAD, BAD],+ ] as const)("%s is %s", (_name, expected, body, header) => {+ expect(agentInteropStatus(req(body, header))).toBe(expected);+ expect(isTrustedAgentInteropRequest(req(body, header))).toBe(+ expected === "trusted",+ );+ });++ it("treats a block without auth as a wrong secret", () => {+ expect(agentInteropStatus({ body: { __agentInterop: {} } })).toBe(+ "invalid",+ );+ });++ it("treats a repeated header as a wrong secret", () => {+ expect(+ agentInteropStatus({ headers: { [AGENT_INTEROP_HEADER]: [GOOD] } }),+ ).toBe("invalid");+ });++ it("treats a missing body or headers as nothing sent", () => {+ expect(agentInteropStatus({})).toBe("none");+ });+});diff --git a/apps/api/src/lib/agent-interop.ts b/apps/api/src/lib/agent-interop.tsindex 66f15647f..102531a00 100644--- a/apps/api/src/lib/agent-interop.ts+++ b/apps/api/src/lib/agent-interop.ts@@ -21,3 +21,32 @@ export function isAgentInteropSecretValid(provided: unknown): boolean {timingSafeEqual(providedBuffer, expectedBuffer));}++export type AgentInteropStatus = "trusted" | "none" | "invalid";++/**+ * Agent-interop standing of the raw request (auth runs before any controller+ * re-parses the body). Each place a secret can be sent (the `__agentInterop`+ * block, AGENT_INTEROP_HEADER) that is present must hold the valid secret:+ * `trusted` needs at least one and no wrong one, `none` means neither is sent.+ */+export function agentInteropStatus(req: {+ body?: any;+ headers?: Record<string, unknown>;+}): AgentInteropStatus {+ const provided: unknown[] = [];+ if (req.body?.__agentInterop != null)+ provided.push(req.body.__agentInterop.auth);+ const headerAuth = req.headers?.[AGENT_INTEROP_HEADER];+ if (headerAuth !== undefined) provided.push(headerAuth);+ if (provided.length === 0) return "none";+ return provided.every(isAgentInteropSecretValid) ? "trusted" : "invalid";+}++/** Presence of the block or header alone is never trusted; only the secret. */+export function isTrustedAgentInteropRequest(req: {+ body?: any;+ headers?: Record<string, unknown>;+}): boolean {+ return agentInteropStatus(req) === "trusted";+}diff --git a/apps/api/src/lib/browser-lifecycle.ts b/apps/api/src/lib/browser-lifecycle.tsindex d7b2e507a..0f73a024a 100644--- a/apps/api/src/lib/browser-lifecycle.ts+++ b/apps/api/src/lib/browser-lifecycle.ts@@ -26,7 +26,10 @@ import {BROWSER_CREDITS_PER_HOUR,INTERACT_CREDITS_PER_HOUR,} from "./browser-billing";-import { getEffectiveConcurrencyLimit } from "./concurrency-limit";+import {+ getEffectiveConcurrencyLimit,+ HOBBY_CONCURRENCY_LIMIT,+} from "./concurrency-limit";import {reserveExternalSlot,mirrorExternalSlotRelease,@@ -53,6 +56,19 @@ export function browserSessionLinks(session: BrowserSessionRow) {};}+/**+ * The 403 every browser entry point returns when the request sent an interop+ * secret that is wrong, or null. Reads the flag authMiddleware set from the+ * raw request, since interact's body parse drops `__agentInterop`.+ */+export function invalidAgentInteropError(+ req: RequestWithAuth<any, any, any>,+): HangarError | null {+ return req.auth.agentInterop === "invalid"+ ? new HangarError(403, "Invalid agent interop.")+ : null;+}+export async function createBrowserSession(req: RequestWithAuth<any, any, any>,options: {@@ -72,14 +88,23 @@ export async function createBrowserSession(503,"Browser feature is not configured (HANGAR_URL is missing).",);+ const invalidInterop = invalidAgentInteropError(req);+ if (invalidInterop) throw invalidInterop;const shouldBill = options.shouldBill ?? true;const estimatedCredits = shouldBill? calculateBrowserSessionCredits(options.ttl * 1000): 0;- const limit = await getEffectiveConcurrencyLimit(+ const teamLimit = await getEffectiveConcurrencyLimit(req.auth.team_id,req.acuc?.org_id ?? null,);+ // An agent run opens browsers against the team's own slots, so a free team+ // (2) is throttled by its own agent. Floor trusted agent traffic at hobby,+ // as the rate limiter does; plans at or above hobby are unchanged.+ const limit =+ req.auth.agentInterop === "trusted"+ ? Math.max(teamLimit, HOBBY_CONCURRENCY_LIMIT)+ : teamLimit;if (shouldBill && req.acuc?.org_id) {const credit = await autumnService.checkCredits({teamId: req.auth.team_id,diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 47da2acab..92b7642ec 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -25,6 +25,13 @@ import { reportPipelineError } from "./redis-pipeline";// Fallback when Autumn can't give us a concurrency value.const DEFAULT_CONCURRENCY_LIMIT = 2;+/**+ * CONCURRENCY granted by the Autumn `hobby` plan (firecrawl-web+ * autumn.config.ts). Pairs with HOBBY_RATE_LIMIT_MULTIPLIER in+ * services/rate-limiter.ts; change both if the hobby plan changes.+ */+export const HOBBY_CONCURRENCY_LIMIT = 5;+/*** Returns the team's effective concurrency limit from Autumn's CONCURRENCY* balance. Autumn is authoritative; when the entity is missing we fall back todiff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex addb9fcd0..b9f8eb78e 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -33,7 +33,10 @@ import { getThirdPartyDataTermsRequiredResponse } from "../lib/exchange";import { getExchangeAccessForRequestBody } from "../lib/exchange-request";import { isToolsOnlySearch } from "../search/alexandria";import { getScrapeZDR } from "../lib/zdr-helpers";-import { isAgentInteropSecretValid } from "../lib/agent-interop";+import {+ agentInteropStatus,+ isAgentInteropSecretValid,+} from "../lib/agent-interop";export function checkCreditsMiddleware(_minimum?: number,@@ -324,7 +327,7 @@ export function authMiddleware(const { team_id, org_id, chunk } = auth;- req.auth = { team_id, org_id };+ req.auth = { team_id, org_id, agentInterop: agentInteropStatus(req) };req.acuc = chunk ?? undefined;next();})().catch(err => next(err));diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.tsindex 92e1682d5..65ef5d4f7 100644--- a/apps/api/src/services/rate-limiter.ts+++ b/apps/api/src/services/rate-limiter.ts@@ -135,7 +135,9 @@ export function getAutumnRateLimiter(* traffic (a valid `__agentInterop` secret) is floored at this multiplier so a* free team's agent runs are limited like hobby rather than at ×1; see* buildAuthenticatedRateLimiter in controllers/auth.ts. Paid plans already- * meet or exceed it, so the floor only ever lifts free.+ * meet or exceed it, so the floor only ever lifts free. Pairs with+ * HOBBY_CONCURRENCY_LIMIT in lib/concurrency-limit.ts; change both if the+ * hobby plan in firecrawl-web autumn.config.ts changes.*/export const HOBBY_RATE_LIMIT_MULTIPLIER = 10;