Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): spend MCP credits first on hosted MCP agent, browser and Alexandria calls (#5463)
Rakshith Ramprakash30cb44503c82
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex f6cef5367..fad9b219a 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -264,6 +264,19 @@ jobs:run: pnpm exec vitest run src/services/billing/__tests__/batch_billing.test.ts --reporter=defaultworking-directory: apps/api+ - name: Run MCP credits billing regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/services/billing/__tests__/credit_billing.test.ts+ src/services/autumn/__tests__/mcp-credits.test.ts+ src/controllers/v2/__tests__/agent-credit-gate.test.ts+ src/controllers/v2/__tests__/browser-mcp-credits.test.ts+ src/services/alexandria/retrieve.test.ts+ src/__tests__/routes/hosted-mcp-agent-interop.test.ts+ --reporter=default+ working-directory: apps/api+- name: Run safeFetch SSRF guard regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: pnpm exec vitest run src/scraper/scrapeURL/engines/utils/safeFetch.proxy.test.ts --reporter=defaultdiff --git a/apps/api/src/__tests__/routes/hosted-mcp-agent-interop.test.ts b/apps/api/src/__tests__/routes/hosted-mcp-agent-interop.test.tsnew file mode 100644index 000000000..8cecd2449--- /dev/null+++ b/apps/api/src/__tests__/routes/hosted-mcp-agent-interop.test.ts@@ -0,0 +1,79 @@+const mocks = vi.hoisted(() => ({ authenticateUser: vi.fn() }));++vi.mock("../../config", async importOriginal => {+ const actual = await importOriginal<typeof import("../../config")>();+ return {+ ...actual,+ config: { ...actual.config, AGENT_INTEROP_SECRET: "agent-secret" },+ };+});+vi.mock("../../controllers/auth", () => ({+ authenticateUser: mocks.authenticateUser,+}));+vi.mock("ioredis", () => ({ default: class {} }));+vi.mock("../../services/redis", () => ({+ getValue: vi.fn(),+ setValue: vi.fn(),+ deleteKey: vi.fn(),+}));++import { authMiddleware } from "../../routes/shared";+import { RateLimiterMode } from "../../types";++async function authOf(+ body: Record<string, unknown>,+ verified: boolean,+ headers: Record<string, string> = {},+) {+ mocks.authenticateUser.mockResolvedValueOnce({+ success: true,+ team_id: "team-1",+ org_id: "org-1",+ chunk: null,+ ...(verified ? { hostedMcp: true } : {}),+ });+ const req: any = { body, headers };+ await new Promise<void>((resolve, reject) =>+ authMiddleware(RateLimiterMode.Scrape)(req, {} as any, (error?: unknown) =>+ error ? reject(error) : resolve(),+ ),+ );+ return req.auth;+}++const interop = (auth: string, hostedMcp?: boolean) => ({+ __agentInterop: {+ auth,+ requestId: "agent-run",+ shouldBill: true,+ ...(hostedMcp === undefined ? {} : { hostedMcp }),+ },+});++describe("hosted MCP signal on agent interop calls", () => {+ beforeEach(() => vi.clearAllMocks());++ it("carries a verified hosted MCP call through", async () => {+ expect(await authOf({}, true)).toMatchObject({ hostedMcp: true });+ });++ it("takes the signal from a trusted interop block", async () => {+ expect(await authOf(interop("agent-secret", true), false)).toMatchObject({+ agentInterop: "trusted",+ hostedMcp: true,+ });+ });++ it.each([+ ["an untrusted interop block", interop("wrong", true), {}],+ ["a trusted block that does not say so", interop("agent-secret"), {}],+ [+ "a trusted header with an untrusted block",+ interop("wrong", true),+ { "x-firecrawl-agent-interop": "agent-secret" },+ ],+ ["a bare flag without interop", { hostedMcp: true }, {}],+ ])("ignores %s", async (_, body, headers) => {+ expect(await authOf(body, false, headers)).not.toHaveProperty("hostedMcp");+ });+});diff --git a/apps/api/src/controllers/v2/__tests__/agent-credit-gate.test.ts b/apps/api/src/controllers/v2/__tests__/agent-credit-gate.test.tsindex c2df6ae01..bc88cb7b5 100644--- a/apps/api/src/controllers/v2/__tests__/agent-credit-gate.test.ts+++ b/apps/api/src/controllers/v2/__tests__/agent-credit-gate.test.ts@@ -2,7 +2,11 @@ vi.mock("../../../lib/logger", () => ({logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },}));vi.mock("../../../services/autumn/autumn.service", () => ({- autumnService: { checkCredits: vi.fn(), hasAutoTopUp: vi.fn() },+ autumnService: {+ checkCredits: vi.fn(),+ hasAutoTopUp: vi.fn(),+ mcpCreditsAvailable: vi.fn(),+ },}));import { gateAgentCredits } from "../agent-credit-gate";@@ -10,6 +14,7 @@ import { autumnService } from "../../../services/autumn/autumn.service";const checkCredits = vi.mocked(autumnService.checkCredits);const hasAutoTopUp = vi.mocked(autumnService.hasAutoTopUp);+const mcpCreditsAvailable = vi.mocked(autumnService.mcpCreditsAvailable);const gate = (requestedMaxCredits?: number, orgId: string | null = "org-1") =>gateAgentCredits({@@ -29,6 +34,7 @@ describe("gateAgentCredits", () => {beforeEach(() => {vi.clearAllMocks();hasAutoTopUp.mockResolvedValue(false);+ mcpCreditsAvailable.mockResolvedValue(1000);});it("blocks a team without overage at a zero balance", async () => {@@ -193,3 +199,160 @@ describe("gateAgentCredits", () => {expect(checkCredits).not.toHaveBeenCalled();});});++describe("gateAgentCredits for a hosted MCP run", () => {+ const hosted = (requestedMaxCredits?: number) =>+ gateAgentCredits({+ teamId: "team-1",+ orgId: "org-1",+ apiKeyId: 7,+ requestedMaxCredits,+ hostedMcp: true,+ });++ beforeEach(() => {+ vi.clearAllMocks();+ hasAutoTopUp.mockResolvedValue(false);+ mcpCreditsAvailable.mockResolvedValue(0);+ });++ it("starts a run MCP credits pay for at a zero balance, capped at them", async () => {+ checkCredits.mockResolvedValueOnce({ allowed: false, remaining: 0 });+ mcpCreditsAvailable.mockResolvedValueOnce(300);++ expect(await hosted()).toEqual({+ allowed: true,+ maxCredits: 300,+ stopOnInsufficientCredits: true,+ });+ expect(mcpCreditsAvailable).toHaveBeenCalledWith(+ expect.objectContaining({+ teamId: "team-1",+ orgId: "org-1",+ value: 2500,+ }),+ );+ });++ it("adds MCP credits to what is left of the balance below the minimum", async () => {+ checkCredits.mockResolvedValueOnce({ allowed: false, remaining: 12 });+ mcpCreditsAvailable.mockResolvedValueOnce(10);++ expect(await hosted(5000)).toEqual({+ allowed: true,+ maxCredits: 22,+ stopOnInsufficientCredits: true,+ });+ });++ it("admits a cap below the minimum when MCP credits clear the minimum", async () => {+ checkCredits.mockResolvedValueOnce({ allowed: false, remaining: 0 });+ mcpCreditsAvailable.mockResolvedValueOnce(20);++ expect(await hosted(10)).toEqual({+ allowed: true,+ maxCredits: 10,+ stopOnInsufficientCredits: true,+ });+ expect(mcpCreditsAvailable).toHaveBeenCalledWith(+ expect.objectContaining({ value: 20 }),+ );+ });++ it("leaves a non-positive maxCredits to the balance alone", async () => {+ checkCredits.mockResolvedValueOnce({ allowed: false, remaining: 0 });+ mcpCreditsAvailable.mockResolvedValue(1000);++ expect(await hosted(-1)).toEqual({ allowed: false, balance: 0 });+ expect(mcpCreditsAvailable).not.toHaveBeenCalled();+ });++ it("refuses when the balance and MCP credits together are below the minimum", async () => {+ checkCredits.mockResolvedValueOnce({ allowed: false, remaining: 12 });+ mcpCreditsAvailable.mockResolvedValueOnce(7);++ expect(await hosted()).toEqual({ allowed: false, balance: 12 });+ });++ it("counts MCP credits alone when a spend limit refused with balance left", async () => {+ checkCredits.mockResolvedValueOnce({ allowed: false, remaining: 5000 });+ mcpCreditsAvailable.mockResolvedValueOnce(19);+ expect(await hosted()).toEqual({ allowed: false, balance: 5000 });++ checkCredits.mockResolvedValueOnce({ allowed: false, remaining: 5000 });+ mcpCreditsAvailable.mockResolvedValueOnce(100);+ expect(await hosted()).toEqual({+ allowed: true,+ maxCredits: 100,+ stopOnInsufficientCredits: true,+ });+ });++ it("raises the balance cap by the MCP credits, up to the request", async () => {+ checkCredits+ .mockResolvedValueOnce({ allowed: true, remaining: 640.6 })+ .mockResolvedValueOnce({ allowed: false, remaining: 640.6 });+ mcpCreditsAvailable.mockResolvedValueOnce(300);+ expect(await hosted()).toEqual({+ allowed: true,+ maxCredits: 940,+ stopOnInsufficientCredits: true,+ });++ checkCredits+ .mockResolvedValueOnce({ allowed: true, remaining: 640 })+ .mockResolvedValueOnce({ allowed: false, remaining: 640 });+ mcpCreditsAvailable.mockResolvedValueOnce(2500);+ expect(await hosted(1000)).toMatchObject({ maxCredits: 1000 });+ });++ it("keeps the balance cap when MCP credits are used up", async () => {+ checkCredits+ .mockResolvedValueOnce({ allowed: true, remaining: 640 })+ .mockResolvedValueOnce({ allowed: false, remaining: 640 });++ expect(await hosted()).toEqual({+ allowed: true,+ maxCredits: 640,+ stopOnInsufficientCredits: true,+ });+ });++ it("starts a capped run MCP credits pay for once the balance is spent", async () => {+ checkCredits+ .mockResolvedValueOnce({ allowed: true, remaining: 0 })+ .mockResolvedValueOnce({ allowed: false, remaining: 0 });+ mcpCreditsAvailable.mockResolvedValueOnce(50);+ expect(await hosted()).toMatchObject({ allowed: true, maxCredits: 50 });++ checkCredits+ .mockResolvedValueOnce({ allowed: true, remaining: 0 })+ .mockResolvedValueOnce({ allowed: false, remaining: 0 });+ expect(await hosted()).toEqual({ allowed: false, balance: 0 });+ });++ it("runs as requested wherever the balance alone already does", async () => {+ checkCredits+ .mockResolvedValueOnce({ allowed: true, remaining: 100 })+ .mockResolvedValueOnce({ allowed: true, remaining: 100 });+ expect(await hosted(8000)).toEqual(asRequested(8000));++ checkCredits+ .mockResolvedValueOnce({ allowed: true, remaining: 100 })+ .mockResolvedValueOnce({ allowed: false, remaining: 100 });+ hasAutoTopUp.mockResolvedValueOnce(true);+ expect(await hosted(4000)).toEqual(asRequested(4000));++ checkCredits.mockResolvedValueOnce(null);+ expect(await hosted(4000)).toEqual(asRequested(4000));+ expect(mcpCreditsAvailable).not.toHaveBeenCalled();+ });++ it("never asks about MCP credits for a run the hosted MCP did not start", async () => {+ checkCredits.mockResolvedValueOnce({ allowed: false, remaining: 0 });+ mcpCreditsAvailable.mockResolvedValue(1000);++ expect(await gate()).toEqual({ allowed: false, balance: 0 });+ expect(mcpCreditsAvailable).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/controllers/v2/__tests__/browser-mcp-credits.test.ts b/apps/api/src/controllers/v2/__tests__/browser-mcp-credits.test.tsnew file mode 100644index 000000000..8ca9fc27d--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/browser-mcp-credits.test.ts@@ -0,0 +1,337 @@+import type { Response } from "express";++const mocks = vi.hoisted(() => ({+ checkCredits: vi.fn(),+ mcpCreditsAvailable: vi.fn(),+ markHostedMcp: vi.fn(),+ cameFromHostedMcp: vi.fn(),+ usedPrompt: vi.fn(),+ settleOnce: vi.fn(),+ billTeam: vi.fn(),+ recordRequestCredits: vi.fn(),+ getHangarBrowser: vi.fn(),+}));++vi.mock("../../../config", async importOriginal => {+ const actual = await importOriginal<typeof import("../../../config")>();+ return {+ ...actual,+ config: { ...actual.config, HANGAR_URL: "http://hangar.test" },+ };+});++vi.mock("../../../lib/logger", () => {+ const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() };+ const child = vi.fn(() => ({ ...log, child }));+ return { logger: { ...log, child } };+});++vi.mock("../../../services/autumn/autumn.service", () => ({+ autumnService: {+ checkCredits: mocks.checkCredits,+ mcpCreditsAvailable: mocks.mcpCreditsAvailable,+ },+ DEFAULT_TEAM_LIMITS: { concurrency_limit: 2 },+}));++vi.mock("../../../services/worker/nuq-router", () => ({+ reserveExternalSlot: vi.fn(async () => true),+ mirrorExternalSlotRelease: vi.fn(async () => {}),+ getCombinedTeamActiveCount: vi.fn(async () => 0),+}));++vi.mock("../../../lib/hangar", async importOriginal => {+ const actual = await importOriginal<typeof import("../../../lib/hangar")>();+ return {+ ...actual,+ createHangarBrowser: vi.fn(async (key: string) => ({+ id: `browser-${key}`,+ cdp_url: "wss://cdp.test",+ view_url: null,+ control_url: null,+ playlist_url: null,+ max_expires_at: null,+ })),+ getHangarBrowser: mocks.getHangarBrowser,+ stopHangarBrowser: vi.fn(async () => {}),+ };+});++vi.mock("../../../lib/browser-sessions", () => ({+ insertBrowserSession: vi.fn(async (row: { id: string }) => ({+ ...row,+ cdp_url: "wss://cdp.test",+ cdp_path: "",+ cdp_interactive_path: "",+ })),+ completeBrowserSessionSettlement: vi.fn(),+ getBrowserSession: vi.fn(),+ getBrowserSessionFromScrape: vi.fn(),+ listBrowserSessions: vi.fn(),+ updateBrowserSessionActivity: vi.fn(),+ updateBrowserSessionScrapeId: vi.fn(),+ settleBrowserSessionOnce: mocks.settleOnce,+ withLockedBrowserSession: vi.fn(async (_id: string, run: any) =>+ run({+ id: "session-1",+ status: "active",+ credits_used: null,+ should_bill: true,+ }),+ ),+ listUnsettledHangarSessions: vi.fn(),+ didBrowserSessionUsePrompt: mocks.usedPrompt,+ markBrowserSessionUsedPrompt: vi.fn(),+ markBrowserSessionHostedMcp: mocks.markHostedMcp,+ didBrowserSessionComeFromHostedMcp: mocks.cameFromHostedMcp,+ upsertBrowserProfile: vi.fn(),+ deleteBrowserProfile: vi.fn(),+}));++vi.mock("../../../services/billing/credit_billing", () => ({+ billTeam: mocks.billTeam,+}));+vi.mock("../../../lib/team-org", () => ({+ orgIdForTeam: vi.fn(async () => "org-1"),+}));+vi.mock("../../../lib/request-credits-store", () => ({+ recordRequestCredits: mocks.recordRequestCredits,+}));+vi.mock("../../../lib/keyless", () => ({+ updateKeylessBrowserCredits: vi.fn(async () => true),+ logKeylessCreditUsage: vi.fn(async () => {}),+ KEYLESS_FREE_TIER_LIMIT_MESSAGE: "limit",+}));+vi.mock("../../../services/logging/log_job", () => ({+ logRequest: vi.fn(async () => {}),+}));++import { browserCreateController } from "../browser";+import {+ reserveBrowserPromptCredits,+ settleBrowserSession,+} from "../../../lib/browser-lifecycle";++const TEAM_ID = "11111111-1111-1111-1111-111111111111";++function makeRes() {+ return {+ statusCode: 200,+ body: undefined as any,+ status(code: number) {+ this.statusCode = code;+ return this;+ },+ json(body: unknown) {+ this.body = body;+ return this;+ },+ };+}++function makeReq(hostedMcp: boolean) {+ return {+ body: { ttl: 600 },+ headers: {},+ path: "/v2/browser",+ auth: {+ team_id: TEAM_ID,+ agentInterop: "none",+ ...(hostedMcp ? { hostedMcp: true } : {}),+ },+ acuc: {+ org_id: "org-1",+ api_key_id: null,+ flags: null,+ concurrency_limit: 10,+ rate_limit_multiplier: 1,+ },+ };+}++async function create(hostedMcp: boolean) {+ const res = makeRes();+ await browserCreateController(+ makeReq(hostedMcp) as any,+ res as unknown as Response,+ );+ return res;+}++const session = {+ id: "session-1",+ team_id: TEAM_ID,+ browser_id: "browser-1",+ request_id: "session-1",+ scrape_id: null,+ status: "active",+ credits_used: null,+ should_bill: true,+ ttl_total: 600,+ zero_data_retention: false,+ profile_name: null,+} as any;++beforeEach(() => {+ vi.clearAllMocks();+ mocks.checkCredits.mockResolvedValue({ allowed: true, remaining: 1000 });+ mocks.mcpCreditsAvailable.mockResolvedValue(0);+ mocks.markHostedMcp.mockResolvedValue(undefined);+ mocks.cameFromHostedMcp.mockResolvedValue(false);+ mocks.usedPrompt.mockResolvedValue(false);+ mocks.billTeam.mockResolvedValue({ success: true });+ mocks.recordRequestCredits.mockResolvedValue(true);+ mocks.getHangarBrowser.mockResolvedValue({ status: "running" });+});++describe("browser sessions from hosted MCP calls", () => {+ it("lets MCP credits pay for a session the balance alone refuses", async () => {+ mocks.checkCredits.mockResolvedValue({ allowed: false, remaining: 5 });+ mocks.mcpCreditsAvailable.mockResolvedValue(15);++ const res = await create(true);+ expect(res.statusCode).toBe(200);+ expect(mocks.mcpCreditsAvailable).toHaveBeenCalledWith(+ expect.objectContaining({ teamId: TEAM_ID, orgId: "org-1", value: 20 }),+ );+ expect(mocks.markHostedMcp).toHaveBeenCalledWith(res.body.id);+ });++ it("refuses a session the balance and MCP credits cannot cover together", async () => {+ mocks.checkCredits.mockResolvedValue({ allowed: false, remaining: 5 });+ mocks.mcpCreditsAvailable.mockResolvedValue(14);++ expect((await create(true)).statusCode).toBe(402);+ expect(mocks.markHostedMcp).not.toHaveBeenCalled();+ });++ it("counts MCP credits alone when a spend limit refused with balance left", async () => {+ mocks.checkCredits.mockResolvedValue({ allowed: false, remaining: 500 });+ mocks.mcpCreditsAvailable.mockResolvedValue(19);+ expect((await create(true)).statusCode).toBe(402);++ mocks.mcpCreditsAvailable.mockResolvedValue(20);+ expect((await create(true)).statusCode).toBe(200);+ });++ it("marks a hosted session even when the balance covers it", async () => {+ const res = await create(true);+ expect(res.statusCode).toBe(200);+ expect(mocks.mcpCreditsAvailable).not.toHaveBeenCalled();+ expect(mocks.markHostedMcp).toHaveBeenCalledWith(res.body.id);+ });++ it("leaves a session not from the hosted MCP as it was", async () => {+ mocks.checkCredits.mockResolvedValue({ allowed: false, remaining: 5 });+ mocks.mcpCreditsAvailable.mockResolvedValue(1000);+ expect((await create(false)).statusCode).toBe(402);++ mocks.checkCredits.mockResolvedValue({ allowed: true, remaining: 1000 });+ expect((await create(false)).statusCode).toBe(200);+ expect(mocks.mcpCreditsAvailable).not.toHaveBeenCalled();+ expect(mocks.markHostedMcp).not.toHaveBeenCalled();+ });++ it("still creates the session when the hosted flag cannot be stored", async () => {+ mocks.markHostedMcp.mockRejectedValue(new Error("redis down"));+ expect((await create(true)).statusCode).toBe(200);+ });++ it("lets MCP credits pay for a prompt the balance alone refuses", async () => {+ mocks.checkCredits.mockResolvedValue({ allowed: false, remaining: 0 });+ mocks.mcpCreditsAvailable.mockResolvedValue(70);+ await expect(+ reserveBrowserPromptCredits(makeReq(true) as any, session),+ ).resolves.toBeUndefined();++ expect(mocks.markHostedMcp).toHaveBeenCalledWith("session-1");++ mocks.mcpCreditsAvailable.mockResolvedValue(69);+ await expect(+ reserveBrowserPromptCredits(makeReq(true) as any, session),+ ).rejects.toMatchObject({ status: 402 });+ await expect(+ reserveBrowserPromptCredits(makeReq(false) as any, session),+ ).rejects.toMatchObject({ status: 402 });+ });++ it("marks the session hosted on a hosted prompt the balance covers", async () => {+ await reserveBrowserPromptCredits(makeReq(true) as any, session);+ expect(mocks.markHostedMcp).toHaveBeenCalledWith("session-1");++ mocks.markHostedMcp.mockClear();+ await reserveBrowserPromptCredits(makeReq(false) as any, session);+ expect(mocks.markHostedMcp).not.toHaveBeenCalled();+ });++ it.each([+ [true, { hostedMcp: true }],+ [false, {}],+ ])(+ "settles a session (hosted: %s) with the hosted MCP signal it was created with",+ async (hosted, extra) => {+ mocks.cameFromHostedMcp.mockResolvedValue(hosted);+ mocks.settleOnce.mockResolvedValue({+ creditsBilled: 10,+ newlySettled: true,+ });+ await settleBrowserSession(session, {+ status: "stopped",+ created_at: 1000,+ ended_at: 1300,+ } as any);+ expect(mocks.billTeam).toHaveBeenCalledWith(TEAM_ID, "org-1", 10, null, {+ endpoint: "browser",+ jobId: "session-1",+ chargeId: "session-1:destroy",+ ...extra,+ });+ },+ );++ it("finishes billing a hosted agent session before settlement returns", async () => {+ mocks.cameFromHostedMcp.mockResolvedValue(true);+ let billed = false;+ let billedBeforeTally = false;+ mocks.billTeam.mockImplementation(async () => {+ await new Promise(resolve => setTimeout(resolve, 20));+ billed = true;+ return { success: true };+ });+ mocks.recordRequestCredits.mockImplementation(async () => {+ billedBeforeTally = billed;+ return true;+ });+ mocks.settleOnce.mockResolvedValue({+ creditsBilled: 10,+ newlySettled: true,+ });+ await settleBrowserSession({ ...session, request_id: "agent-run" }, {+ status: "stopped",+ created_at: 1000,+ ended_at: 1300,+ } as any);+ expect(billed).toBe(true);+ expect(billedBeforeTally).toBe(true);+ });++ it("names the agent run on an agent session's charge", async () => {+ mocks.cameFromHostedMcp.mockResolvedValue(true);+ mocks.settleOnce.mockResolvedValue({+ creditsBilled: 10,+ newlySettled: true,+ });+ await settleBrowserSession({ ...session, request_id: "agent-run" }, {+ status: "stopped",+ created_at: 1000,+ ended_at: 1300,+ } as any);+ expect(mocks.billTeam).toHaveBeenCalledWith(TEAM_ID, "org-1", 10, null, {+ endpoint: "agent",+ jobId: "agent-run",+ chargeId: "session-1:destroy",+ agentRequestId: "agent-run",+ hostedMcp: true,+ });+ });+});diff --git a/apps/api/src/controllers/v2/agent-credit-gate.ts b/apps/api/src/controllers/v2/agent-credit-gate.tsindex 5627192b4..a7aba51a3 100644--- a/apps/api/src/controllers/v2/agent-credit-gate.ts+++ b/apps/api/src/controllers/v2/agent-credit-gate.ts@@ -28,6 +28,10 @@ export type AgentCreditGate =* - otherwise: `maxCredits` is capped at the balance, and the agent service* is told to stop the run if the balance runs out mid-flight.*+ * A hosted MCP run counts the team's MCP credits on top of its balance, since+ * its charges spend them first: they can admit a run the balance alone would+ * refuse, and raise the cap, but never lower anything the balance allows.+ ** Fails open: whenever Autumn or firebill can't answer, the run starts as* requested.*/@@ -36,11 +40,13 @@ export async function gateAgentCredits({orgId,apiKeyId,requestedMaxCredits,+ hostedMcp = false,}: {teamId: string;orgId: string | null | undefined;apiKeyId: number | null;requestedMaxCredits: number | undefined;+ hostedMcp?: boolean;}): Promise<AgentCreditGate> {const asRequested: AgentCreditGate = {allowed: true,@@ -57,9 +63,51 @@ export async function gateAgentCredits({properties: { source, path: "/agent", apiKeyId },});+ const requested = requestedMaxCredits ?? AGENT_DEFAULT_MAX_CREDITS;+ const mcpCredits = () =>+ hostedMcp && requested > 0+ ? autumnService.mcpCreditsAvailable({+ teamId,+ orgId,+ // Never below the minimum, which a smaller cap still has to clear.+ value: Math.max(requested, AGENT_MIN_CREDITS),+ properties: {+ source: "agentCreditGate:mcp",+ path: "/agent",+ apiKeyId,+ },+ })+ : Promise.resolve(0);+ // Capped at what the balance and the MCP credits can pay together, and+ // stopped mid-run once they can't cover it.+ const capped = (balance: number, mcp: number): AgentCreditGate => {+ const maxCredits = Math.min(requested, Math.max(balance, 0) + mcp);+ if (maxCredits < requested) {+ logger.info("Capping agent maxCredits at the team's balance", {+ teamId,+ requestedMaxCredits,+ maxCredits,+ mcpCredits: mcp,+ });+ }+ return { allowed: true, maxCredits, stopOnInsufficientCredits: true };+ };+const minimum = await check(AGENT_MIN_CREDITS, "agentCreditGate");if (minimum === null) return asRequested;- if (!minimum.allowed) return { allowed: false, balance: minimum.remaining };+ if (!minimum.allowed) {+ // What is left of the balance counts only when the balance is why the+ // check refused; a spend limit refuses with plenty left, and the MCP+ // credits alone may still pay for the run.+ const fromBalance =+ minimum.remaining < AGENT_MIN_CREDITS+ ? Math.max(Math.floor(minimum.remaining), 0)+ : 0;+ const mcp = await mcpCredits();+ if (fromBalance + mcp < AGENT_MIN_CREDITS)+ return { allowed: false, balance: minimum.remaining };+ return capped(fromBalance, mcp);+ }// An unlimited balance comes back as int64 max. Probing past it overflows// the billing service's integer, so treat it as unlimited and skip the probe.@@ -72,16 +120,7 @@ export async function gateAgentCredits({);if (pastBalance === null || pastBalance.allowed) return asRequested;if (await autumnService.hasAutoTopUp(orgId)) return asRequested;- if (balance <= 0) return { allowed: false, balance };-- const requested = requestedMaxCredits ?? AGENT_DEFAULT_MAX_CREDITS;- const maxCredits = Math.min(requested, balance);- if (maxCredits < requested) {- logger.info("Capping agent maxCredits at the team's balance", {- teamId,- requestedMaxCredits,- maxCredits,- });- }- return { allowed: true, maxCredits, stopOnInsufficientCredits: true };+ const mcp = await mcpCredits();+ if (balance <= 0 && mcp <= 0) return { allowed: false, balance };+ return capped(balance, mcp);}diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex 0b7a59498..a0a291c23 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -1,5 +1,8 @@import { v7 as uuidv7 } from "uuid";-import { AGENT_REQUEST_CREDITS_SHARDS } from "../../lib/request-credits-store";+import {+ AGENT_REQUEST_CREDITS_SHARDS,+ initializeRequestCredits,+} from "../../lib/request-credits-store";import { Response } from "express";import {AgentRequest,@@ -25,6 +28,7 @@ import {import { UnsafeDomainBlockedError } from "../../lib/threat-protection/error";import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";+import { agentMcpCreditsRequestId } from "../../services/billing/agent-mcp-credits";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";import { fetchAgentThread, threadErrorFor } from "./agent-thread";import {@@ -142,7 +146,12 @@ export async function agentController(req.acuc?.org_id ?? null,threatScanCredits,req.acuc?.api_key_id ?? null,- { endpoint: "agent", jobId: agentId, chargeId: `${agentId}:threat` },+ {+ endpoint: "agent",+ jobId: agentId,+ chargeId: `${agentId}:threat`,+ hostedMcp: req.auth.hostedMcp,+ },).catch(error => {logger.error(`Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,@@ -254,6 +263,7 @@ export async function agentController(orgId: req.auth.org_id,apiKeyId: req.acuc?.api_key_id ?? null,requestedMaxCredits: req.body.maxCredits,+ hostedMcp: req.auth.hostedMcp,});if (!gate.allowed) {logger.warn("Insufficient credits to start agent run", {@@ -288,6 +298,19 @@ export async function agentController(: {}),});+ // The tally a hosted MCP run's calls add their MCP credits to, so a failed+ // run can give that part back to MCP credits (see tallyAgentMcpCredits).+ if (req.auth.hostedMcp) {+ await initializeRequestCredits(+ agentMcpCreditsRequestId(agentId),+ AGENT_REQUEST_CREDITS_SHARDS,+ ).catch(error =>+ logger.error("Failed to initialize the agent run's MCP credits tally", {+ error,+ }),+ );+ }+const passthrough = await fetch(config.EXTRACT_V3_BETA_URL + "/internal/extracts",{@@ -317,6 +340,7 @@ export async function agentController(exchange: req.body.exchange,location: req.body.location,safe: req.body.safe,+ ...(req.auth.hostedMcp ? { hostedMcp: true } : {}),...(zeroDataRetention ? { zeroDataRetention: true } : {}),...(zeroDataRetentionMinutes !== null? { zeroDataRetentionMinutes }diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 285ac3fea..1b6b2795b 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -147,7 +147,11 @@ export async function batchScrapeController(const id = req.body.appendToId ?? uuidv7();const billing: BillingMetadata = {...(req.body.__agentInterop- ? { endpoint: "agent" as const, jobId: id }+ ? {+ endpoint: "agent" as const,+ jobId: id,+ agentRequestId: req.body.__agentInterop.requestId,+ }: { endpoint: "batch_scrape" as const, jobId: id }),externalRequestId: externalRequestId(req),hostedMcp: req.auth.hostedMcp,@@ -301,7 +305,7 @@ export async function batchScrapeController(threatScanCredits > 0 &&(req.body.__agentInterop?.shouldBill ?? true)) {- billTeam(+ const charge = billTeam(req.auth.team_id,req.acuc?.org_id ?? null,threatScanCredits,@@ -318,6 +322,8 @@ export async function batchScrapeController(`Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,);});+ // See search: a hosted MCP agent run's MCP credits part lands first.+ if (billing.hostedMcp && billing.agentRequestId) await charge;}emitRejectedScrapeActivityEvents(blocked.map(blockedUrl => ({diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex b7c96fbd6..7c1219aeb 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -57,6 +57,7 @@ export const browserCreateRequestSchema = z.object({auth: z.string(),requestId: z.string().uuid(),shouldBill: z.boolean(),+ hostedMcp: z.boolean().optional(),}).optional(),});diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex 6ea287874..6090959a6 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -372,7 +372,11 @@ export async function parseController((req.body.zeroDataRetention ?? false);const billing: BillingMetadata = {...(req.body.__agentInterop- ? { endpoint: "agent" as const, jobId }+ ? {+ endpoint: "agent" as const,+ jobId,+ agentRequestId: req.body.__agentInterop.requestId,+ }: { endpoint: "parse" as const, jobId }),externalRequestId: externalRequestId(req),hostedMcp: req.auth.hostedMcp,diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex ac505dc71..b35f1f34a 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -52,6 +52,7 @@ const agentInteropSchema = z.strictObject({requestId: z.string(),shouldBill: z.boolean(),boostConcurrency: z.boolean().optional(),+ hostedMcp: z.boolean().optional(),});const providerScrapeSchema = z.strictObject({@@ -173,6 +174,7 @@ export async function providerScrapeController(resultAuthorization: req.get("authorization"),requestId,agentRequestId: body.__agentInterop?.requestId,+ hostedMcp: req.auth.hostedMcp,scrapeId: uuidv7(),timeoutMs: body.timeout,bypassBilling: body.__agentInterop?.shouldBill === false,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 1f89cd99a..a18555631 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -222,7 +222,11 @@ export async function scrapeController(});const billing: BillingMetadata = {...(req.body.__agentInterop- ? { endpoint: "agent" as const, jobId }+ ? {+ endpoint: "agent" as const,+ jobId,+ agentRequestId: req.body.__agentInterop.requestId,+ }: { endpoint: "scrape" as const, jobId }),externalRequestId: externalRequestId(req),hostedMcp: req.auth.hostedMcp,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 6f43f2d41..e2dc76a90 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -312,7 +312,11 @@ async function searchControllerInner(const agentRequestId = req.body.__agentInterop?.requestId ?? null;const billing: BillingMetadata = {...(req.body.__agentInterop- ? { endpoint: "agent" as const, jobId }+ ? {+ endpoint: "agent" as const,+ jobId,+ agentRequestId: req.body.__agentInterop.requestId,+ }: { endpoint: "search" as const, jobId }),externalRequestId: externalRequestId(req),hostedMcp: req.auth.hostedMcp,@@ -523,7 +527,7 @@ async function searchControllerInner(// Bill team for search credits only (scrape jobs bill themselves)if (!isSearchPreview && shouldBill) {- billTeam(+ const charge = billTeam(req.auth.team_id,req.acuc?.org_id ?? null,result.searchCredits,@@ -536,6 +540,9 @@ async function searchControllerInner(error,}),);+ // A hosted MCP agent run's call lands its MCP credits part before it+ // answers, so the run never reads its credits without it.+ if (billing.hostedMcp && billing.agentRequestId) await charge;}if (reservedKeylessCredits > 0) {diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex ab9df62cc..aa57549c3 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1239,6 +1239,7 @@ const scrapeRequestSchemaBase = baseScrapeOptions.safeExtend({requestId: z.string(),shouldBill: z.boolean(),boostConcurrency: z.boolean().optional(),+ hostedMcp: z.boolean().optional(),}).optional(),});@@ -1296,6 +1297,7 @@ const parseRequestSchemaBase = baseScrapeOptions.safeExtend({requestId: z.string(),shouldBill: z.boolean(),boostConcurrency: z.boolean().optional(),+ hostedMcp: z.boolean().optional(),}).optional(),file: uploadedParseFileSchema,@@ -1332,6 +1334,7 @@ const batchScrapeRequestSchemaBase = baseScrapeOptions.safeExtend({auth: z.string(),requestId: z.string(),shouldBill: z.boolean(),+ hostedMcp: z.boolean().optional(),}).optional(),});@@ -1357,6 +1360,7 @@ const batchScrapeRequestSchemaNoURLValidationBase =auth: z.string(),requestId: z.string(),shouldBill: z.boolean(),+ hostedMcp: z.boolean().optional(),}).optional(),});@@ -2121,7 +2125,8 @@ type AuthObject = {// the body and drop `__agentInterop`, so read this instead.agentInterop?: AgentInteropStatus;// Set only by authMiddleware, from the hosted MCP verification in- // supaAuthenticateUser; never from the request body or headers directly.+ // supaAuthenticateUser, or from a trusted agent interop block for an agent+ // run that verification started; never from an untrusted body or header.hostedMcp?: boolean;};@@ -2693,6 +2698,7 @@ export const searchRequestSchema = zauth: z.string(),requestId: z.string(),shouldBill: z.boolean(),+ hostedMcp: z.boolean().optional(),}).optional(),})diff --git a/apps/api/src/lib/browser-lifecycle.ts b/apps/api/src/lib/browser-lifecycle.tsindex 43fdb4ace..de66ca72f 100644--- a/apps/api/src/lib/browser-lifecycle.ts+++ b/apps/api/src/lib/browser-lifecycle.ts@@ -19,6 +19,8 @@ import {settleBrowserSessionOnce,withLockedBrowserSession,didBrowserSessionUsePrompt,+ didBrowserSessionComeFromHostedMcp,+ markBrowserSessionHostedMcp,listUnsettledHangarSessions,BrowserSessionRow,} from "./browser-sessions";@@ -183,7 +185,16 @@ async function createBrowserSessionInternal(apiKeyId: req.acuc?.api_key_id ?? null,},});- if (credit !== null && !credit.allowed)+ if (+ credit !== null &&+ !credit.allowed &&+ !(await mcpCreditsCover(+ req,+ credit.remaining,+ estimatedCredits,+ "browserCreate:mcp",+ ))+ )throw new BrowserSessionError(402,`Insufficient credits for a ${options.ttl}s browser session (requires ~${estimatedCredits} credits).`,@@ -251,6 +262,15 @@ async function createBrowserSessionInternal(credits_used: null,profile_name: options.profile?.name ?? null,});+ // Settlement runs later, often with no request in hand, so the session+ // keeps the verified hosted MCP signal for it.+ if (shouldBill && req.auth.hostedMcp)+ await markBrowserSessionHostedMcp(session.id).catch(error =>+ logger.warn("Failed to mark a hosted MCP browser session", {+ sessionId: session.id,+ error,+ }),+ );return {session,expiresAt:@@ -277,6 +297,31 @@ async function createBrowserSessionInternal(}}+/**+ * Whether a hosted MCP call's MCP credits pay for what its balance refused.+ * What is left of the balance counts only when the balance is why the check+ * refused; a spend limit refuses with plenty left.+ */+async function mcpCreditsCover(+ req: RequestWithAuth<any, any, any>,+ remaining: number,+ value: number,+ source: string,+): Promise<boolean> {+ if (!req.auth.hostedMcp || !req.acuc?.org_id) return false;+ const mcp = await autumnService.mcpCreditsAvailable({+ teamId: req.auth.team_id,+ orgId: req.acuc.org_id,+ value,+ properties: {+ source,+ path: req.path,+ apiKeyId: req.acuc.api_key_id ?? null,+ },+ });+ return (remaining < value ? Math.max(remaining, 0) : 0) + mcp >= value;+}+export async function settleBrowserSession(session: BrowserSessionRow,browser: HangarBrowser,@@ -341,7 +386,8 @@ async function settleBrowserSessionInternal(? session.request_id: undefined;if (session.should_bill) {- billTeam(+ const hostedMcp = await didBrowserSessionComeFromHostedMcp(session.id);+ const charge = billTeam(session.team_id,await orgIdForTeam(session.team_id),creditsBilled,@@ -354,6 +400,8 @@ async function settleBrowserSessionInternal(: "browser",jobId: agentRequestId ?? session.id,chargeId: `${session.id}:destroy`,+ ...(agentRequestId ? { agentRequestId } : {}),+ ...(hostedMcp ? { hostedMcp: true } : {}),},).catch(error =>logger.error("Failed to bill browser session", {@@ -362,6 +410,10 @@ async function settleBrowserSessionInternal(error,}),);+ // A hosted MCP agent run's charge lands, MCP credits part included,+ // before the run's credits below, so the run never reads one without+ // the other.+ if (hostedMcp && agentRequestId) await charge;}if (agentRequestId) {await recordRequestCredits({@@ -421,7 +473,16 @@ export async function reserveBrowserPromptCredits(apiKeyId: req.acuc?.api_key_id ?? null,},});- if (credit !== null && !credit.allowed)+ if (+ credit !== null &&+ !credit.allowed &&+ !(await mcpCreditsCover(+ req,+ credit.remaining,+ credits,+ "browserPrompt:mcp",+ ))+ )throw new BrowserSessionError(402,"Insufficient credits for a browser prompt session.",@@ -440,6 +501,15 @@ export async function reserveBrowserPromptCredits()throw new BrowserSessionError(429, KEYLESS_FREE_TIER_LIMIT_MESSAGE);await markBrowserSessionUsedPrompt(current.id);+ // The prompt rate is most of a session's charge, so a hosted MCP prompt+ // bills the session as hosted even when another call created it.+ if (current.should_bill && req.auth.hostedMcp)+ await markBrowserSessionHostedMcp(current.id).catch(error =>+ logger.warn("Failed to mark a hosted MCP browser session", {+ sessionId: current.id,+ error,+ }),+ );});}diff --git a/apps/api/src/lib/browser-sessions.ts b/apps/api/src/lib/browser-sessions.tsindex 96424b397..0b0be4cf7 100644--- a/apps/api/src/lib/browser-sessions.ts+++ b/apps/api/src/lib/browser-sessions.ts@@ -6,6 +6,7 @@ import { redisRateLimitClient } from "../services/rate-limiter";import { db } from "../db/connection";import * as schema from "../db/schema";import { browserProfileDeletedKey } from "./browser-profiles";+import { logger } from "./logger";type BrowserSessionStatus = "active" | "destroyed" | "error";@@ -352,3 +353,31 @@ export async function didBrowserSessionUsePrompt(): Promise<boolean> {return (await getValue(promptFlagKey(sessionId))) === "1";}++// ---------------------------------------------------------------------------+// Hosted MCP sessions (Redis)+// ---------------------------------------------------------------------------++function hostedMcpFlagKey(sessionId: string): string {+ return `browser_session:hosted_mcp:${sessionId}`;+}++export async function markBrowserSessionHostedMcp(+ sessionId: string,+): Promise<void> {+ await setValue(hostedMcpFlagKey(sessionId), "1", PROFILE_DELETED_TTL_SECONDS);+}++/** A flag lost to Redis bills the session as before: on regular credits. */+export async function didBrowserSessionComeFromHostedMcp(+ sessionId: string,+): Promise<boolean> {+ const flag = await getValue(hostedMcpFlagKey(sessionId)).catch(error => {+ logger.warn("Failed to read a browser session's hosted MCP flag", {+ sessionId,+ error,+ });+ return null;+ });+ return flag === "1";+}diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex f904326a9..b5b15922c 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -345,12 +345,20 @@ export function authMiddleware(}}- const { team_id, org_id, chunk, hostedMcp } = auth;+ const { team_id, org_id, chunk } = auth;+ const agentInterop = agentInteropStatus(req);+ // An agent run started by a hosted MCP call makes its own calls under+ // the interop secret, and says so in the interop block. Only a trusted+ // block may say it: the run itself was verified when it started.+ const hostedMcp =+ auth.hostedMcp ||+ (agentInterop === "trusted" &&+ (req.body as any)?.__agentInterop?.hostedMcp === true);req.auth = {team_id,org_id,- agentInterop: agentInteropStatus(req),+ agentInterop,...(hostedMcp ? { hostedMcp: true } : {}),};req.acuc = chunk ?? undefined;diff --git a/apps/api/src/services/alexandria/retrieve.test.ts b/apps/api/src/services/alexandria/retrieve.test.tsindex 0d8620955..d099f3ffc 100644--- a/apps/api/src/services/alexandria/retrieve.test.ts+++ b/apps/api/src/services/alexandria/retrieve.test.ts@@ -17,6 +17,8 @@ const mocks = vi.hoisted(() => {lock: vi.fn(),finalize: vi.fn(),refund: vi.fn(),+ mcpAvailable: vi.fn(),+ refundMcp: vi.fn(),billAdd: vi.fn(),report: vi.fn(),initCredits: vi.fn(),@@ -43,9 +45,12 @@ vi.mock("../autumn/autumn.service", () => ({lockCredits: mocks.lock,finalizeCreditsLock: mocks.finalize,refundCredits: mocks.refund,+ mcpCreditsAvailable: mocks.mcpAvailable,+ refundHostedMcpCredits: mocks.refundMcp,isRoutedThroughFirebill: async () => false,},featureIdForBillingEndpoint: () => "credits",+ MCP_CREDITS_FEATURE_ID: "mcp_credits",}));import { retrieveProviders } from "./retrieve";@@ -91,6 +96,8 @@ beforeEach(() => {mocks.lock.mockResolvedValue({ status: "locked", lockId: "held" });mocks.finalize.mockResolvedValue(true);mocks.refund.mockResolvedValue(undefined);+ mocks.mcpAvailable.mockResolvedValue(0);+ mocks.refundMcp.mockResolvedValue(true);mocks.billAdd.mockResolvedValue({});mocks.report.mockResolvedValue(true);mocks.initCredits.mockResolvedValue(true);@@ -754,3 +761,195 @@ it("does not retry the write when the credit store is unavailable", async () =>});expect(mocks.recordCredits).toHaveBeenCalledTimes(1);});++describe("hosted MCP calls", () => {+ const hosted = (overrides: Record<string, unknown> = {}) =>+ run({ hostedMcp: true, ...overrides });+ const locks = () => mocks.lock.mock.calls.map(([arg]) => arg);+ const settles = () => mocks.finalize.mock.calls.map(([arg]) => arg);+ const billing = () => mocks.billAdd.mock.calls[0][1].billing;++ beforeEach(() => {+ mocks.lock.mockImplementation(async arg => ({+ status: "locked",+ lockId: arg.lockId,+ }));+ });++ it("holds and settles on the MCP credits alone when they cover the call", async () => {+ mocks.mcpAvailable.mockResolvedValue(5);+ expect((await hosted()).status).toBe(200);+ expect(mocks.mcpAvailable).toHaveBeenCalledWith(+ expect.objectContaining({ teamId: "team", orgId: "org", value: 5 }),+ );+ expect(locks()).toEqual([+ expect.objectContaining({+ value: 5,+ featureId: "mcp_credits",+ lockId: expect.stringMatching(/_mcp$/),+ }),+ ]);+ expect(settles()).toEqual([+ expect.objectContaining({+ lockId: locks()[0].lockId,+ action: "confirm",+ overrideValue: 3,+ heldValue: 5,+ featureId: "mcp_credits",+ }),+ ]);+ expect(billing()).toEqual(+ expect.objectContaining({ hostedMcp: true, mcpCredits: 3 }),+ );+ });++ it("splits the hold and the settle, MCP credits first, when they cover part", async () => {+ mocks.mcpAvailable.mockResolvedValue(2);+ expect((await hosted()).status).toBe(200);+ expect(locks()).toEqual([+ expect.objectContaining({ value: 2, featureId: "mcp_credits" }),+ expect.objectContaining({ value: 3, featureId: "credits" }),+ ]);+ expect(settles()).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ featureId: "mcp_credits",+ action: "confirm",+ overrideValue: 2,+ heldValue: 2,+ }),+ expect.objectContaining({+ featureId: "credits",+ action: "confirm",+ overrideValue: 1,+ heldValue: 3,+ }),+ ]),+ );+ expect(settles()).toHaveLength(2);+ expect(billing()).toEqual(+ expect.objectContaining({ hostedMcp: true, mcpCredits: 2 }),+ );+ });++ it("releases the regular hold when the MCP part pays the whole charge", async () => {+ mocks.mcpAvailable.mockResolvedValue(4);+ expect((await hosted()).status).toBe(200);+ expect(settles()).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ featureId: "mcp_credits",+ overrideValue: 3,+ heldValue: 4,+ }),+ expect.objectContaining({ featureId: "credits", action: "release" }),+ ]),+ );+ expect(billing()).toEqual(expect.objectContaining({ mcpCredits: 3 }));+ });++ it("holds and settles on regular credits alone once the MCP credits are used up", async () => {+ expect((await hosted()).status).toBe(200);+ expect(locks()).toEqual([+ expect.objectContaining({ value: 5, featureId: "credits" }),+ ]);+ expect(settles()).toEqual([+ expect.objectContaining({ featureId: "credits", overrideValue: 3 }),+ ]);+ expect(billing()).not.toHaveProperty("mcpCredits");+ });++ it("hands the MCP hold back when the rest cannot be held", async () => {+ mocks.mcpAvailable.mockResolvedValue(2);+ mocks.lock+ .mockImplementationOnce(async arg => ({+ status: "locked",+ lockId: arg.lockId,+ }))+ .mockResolvedValueOnce({ status: "denied" });+ const refused = await hosted();+ expect(refused.status).toBe(402);+ expect(executions()).toHaveLength(0);+ expect(settles()).toEqual([+ expect.objectContaining({+ featureId: "mcp_credits",+ action: "release",+ heldValue: 2,+ }),+ ]);+ });++ it.each([+ ["the MCP part", "mcp_credits", 1, 0],+ ["the rest", "credits", 2, 2],+ ])(+ "records only what landed when %s does not settle",+ async (_, failing, credits, mcpCredits) => {+ mocks.mcpAvailable.mockResolvedValue(2);+ mocks.finalize.mockImplementation(async arg => arg.featureId !== failing);+ expect((await hosted({ agentRequestId: "agent-run" })).status).toBe(200);+ expect(mocks.billAdd).toHaveBeenCalledWith(+ "bill_team",+ expect.objectContaining({+ credits,+ billing: expect.objectContaining(+ mcpCredits > 0 ? { mcpCredits } : {},+ ),+ }),+ expect.anything(),+ );+ if (mcpCredits === 0) expect(billing()).not.toHaveProperty("mcpCredits");+ expect(mocks.recordCredits).toHaveBeenCalledWith(+ expect.objectContaining({ requestId: "agent-run", credits }),+ );+ expect(mocks.report).not.toHaveBeenCalled();+ },+ );++ it("releases both holds on a definitive refusal", async () => {+ mocks.mcpAvailable.mockResolvedValue(2);+ exchangeAnswers({ error: "nope" }, 400);+ expect((await hosted()).status).toBe(400);+ expect(settles()).toHaveLength(2);+ expect(settles().every(settle => settle.action === "release")).toBe(true);+ });++ it("refunds each part to its own balance when the ledger enqueue fails", async () => {+ mocks.mcpAvailable.mockResolvedValue(2);+ mocks.billAdd.mockRejectedValue(new Error("queue down"));+ expect((await hosted()).status).toBe(200);+ expect(mocks.refundMcp).toHaveBeenCalledWith(+ expect.objectContaining({ value: 3, mcpCredits: 2, orgId: "org" }),+ );+ expect(mocks.refund).not.toHaveBeenCalled();+ });++ it("adds the MCP part of an agent call to the run's MCP tally", async () => {+ mocks.mcpAvailable.mockResolvedValue(2);+ await hosted({ agentRequestId: "agent-run" });+ expect(mocks.recordCredits).toHaveBeenCalledWith({+ requestId: "agent-run",+ jobId: expect.stringMatching(/^alexandria:/),+ credits: 3,+ });+ expect(mocks.recordCredits).toHaveBeenCalledWith({+ requestId: "agent-run:mcp",+ jobId: expect.stringMatching(/^alexandria:/),+ credits: 2,+ });+ });++ it("leaves a call not from the hosted MCP as it was", async () => {+ mocks.mcpAvailable.mockResolvedValue(5);+ expect((await run({ agentRequestId: "agent-run" })).status).toBe(200);+ expect(mocks.mcpAvailable).not.toHaveBeenCalled();+ expect(locks()).toEqual([+ expect.objectContaining({ value: 5, featureId: "credits" }),+ ]);+ expect(billing()).toEqual({+ endpoint: "scrape",+ chargeId: expect.any(String),+ });+ expect(mocks.recordCredits).toHaveBeenCalledTimes(1);+ });+});diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex 33586922d..4110e6303 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -12,7 +12,9 @@ import { logger } from "../../lib/logger";import {autumnService,featureIdForBillingEndpoint,+ MCP_CREDITS_FEATURE_ID,} from "../autumn/autumn.service";+import { tallyAgentMcpCredits } from "../billing/agent-mcp-credits";import type { LockCreditsResult } from "../autumn/types";import {AGENT_REQUEST_CREDITS_SHARDS,@@ -52,6 +54,10 @@ type Retrieval = {scrapeId: string;lockId?: string;maximumCredits?: number;+ // The part of the hold a hosted MCP call's MCP credits took, under its own+ // lock; the rest of maximumCredits is held under lockId.+ mcpLockId?: string;+ mcpHeldCredits?: number;response?: ExchangeResponse;// Set instead of response under zero data retention.responseWithheld?: boolean;@@ -107,6 +113,8 @@ export async function retrieveProviders(input: {requestId: string;/** The agent run this call belongs to: its credits join that run's call credits. */agentRequestId?: string;+ /** A verified hosted MCP call: its MCP credits are held and spent first. */+ hostedMcp?: boolean;scrapeId: string;timeoutMs: number;bypassBilling?: boolean;@@ -275,10 +283,16 @@ export async function retrieveProviders(input: {};let lockId: string | undefined;+ let mcpLockId: string | undefined;+ let mcpHeldCredits = 0;let maximumCredits: number;let refundable = false;- const finalize = async (credits: number) => {- if (!lockId) return true;+ const settle = async (+ lock: string,+ heldValue: number,+ credits: number,+ lockFeatureId: string,+ ) => {const settled = await autumnService.finalizeCreditsLock({// Omitted without an org: the settle goes straight to Autumn, the route// an unnameable org already took.@@ -286,22 +300,46 @@ export async function retrieveProviders(input: {input.orgId !== null? { teamId: input.teamId, orgId: input.orgId }: undefined,- lockId,+ lockId: lock,action: credits > 0 ? "confirm" : "release",...(credits > 0 ? { overrideValue: credits } : {}),- heldValue: maximumCredits,- featureId,+ heldValue,+ featureId: lockFeatureId,properties,});if (!settled)logger.error("Provider credits were not settled; hold expires unbilled", {chargeId: id,teamId: input.teamId,- lockId,+ lockId: lock,credits,});return settled;};+ // The MCP credits pay first, up to what they hold; the rest settles on the+ // scrape feature's hold.+ const mcpPart = (credits: number) => Math.min(credits, mcpHeldCredits);+ const finalizeParts = (credits: number) =>+ Promise.all([+ mcpLockId+ ? settle(+ mcpLockId,+ mcpHeldCredits,+ mcpPart(credits),+ MCP_CREDITS_FEATURE_ID,+ )+ : true,+ lockId+ ? settle(+ lockId,+ maximumCredits - mcpHeldCredits,+ credits - mcpPart(credits),+ featureId,+ )+ : true,+ ]);+ const finalize = async (credits: number) =>+ (await finalizeParts(credits)).every(Boolean);try {const denied = await authorizeProviders(@@ -384,16 +422,45 @@ export async function retrieveProviders(input: {input.teamId,input.orgId,));- hold = await autumnService.lockCredits({- teamId: input.teamId,- orgId: input.orgId,- value: maximumCredits,- lockId: `alexandria_${id}`,- expiresAt: record.deadline + HOLD_GRACE_MS,- featureId,- properties,- });+ if (input.hostedMcp) {+ const mcpCredits = await autumnService.mcpCreditsAvailable({+ teamId: input.teamId,+ orgId: input.orgId,+ value: maximumCredits,+ properties,+ });+ const mcpHold =+ mcpCredits > 0+ ? await autumnService.lockCredits({+ teamId: input.teamId,+ orgId: input.orgId,+ value: mcpCredits,+ lockId: `alexandria_${id}_mcp`,+ expiresAt: record.deadline + HOLD_GRACE_MS,+ featureId: MCP_CREDITS_FEATURE_ID,+ properties,+ })+ : undefined;+ if (mcpHold?.status === "locked") {+ mcpLockId = mcpHold.lockId;+ mcpHeldCredits = mcpCredits;+ }+ }+ // Held in full on the MCP credits, nothing is left to hold here.+ hold =+ mcpHeldCredits < maximumCredits+ ? await autumnService.lockCredits({+ teamId: input.teamId,+ orgId: input.orgId,+ value: maximumCredits - mcpHeldCredits,+ lockId: `alexandria_${id}`,+ expiresAt: record.deadline + HOLD_GRACE_MS,+ featureId,+ properties,+ })+ : { status: "locked", lockId: "" };}+ if (hold.status !== "locked" && mcpLockId) await finalize(0);if (hold.status === "denied")return refuse(hold.reason === "gate_unavailable"@@ -415,16 +482,20 @@ export async function retrieveProviders(input: {"Credit reservation is unavailable. No provider was executed.",),);- lockId = hold.lockId;+ lockId = hold.lockId || undefined;}// The record is written before the provider runs, so a failed write means// nothing ran: the catch below hands back the index calls and the hold.record.lockId = lockId;record.maximumCredits = maximumCredits;+ if (mcpLockId) {+ record.mcpLockId = mcpLockId;+ record.mcpHeldCredits = mcpHeldCredits;+ }await write(record);} catch (error) {- if (lockId) await finalize(0).catch(() => false);+ if (lockId || mcpLockId) await finalize(0).catch(() => false);await releaseIndexCalls();await redisRateLimitClient.del(key).catch(() => {});throw error;@@ -496,10 +567,16 @@ export async function retrieveProviders(input: {if (!receiptMatches)return fail("Alexandria receipt did not match the request");- const credits = answer.creditsCost;- const settled = await finalize(credits);+ const [mcpSettled, restSettled] = await finalizeParts(answer.creditsCost);+ const settled = mcpSettled && restSettled;+ // A split settle that landed in part records the part that landed; the+ // rest goes unbilled, as a whole settle that does not land does.+ const mcpCredits = mcpSettled ? mcpPart(answer.creditsCost) : 0;+ const credits =+ mcpCredits ++ (restSettled ? answer.creditsCost - mcpPart(answer.creditsCost) : 0);const recorded =- settled && billable && credits > 0+ billable && credits > 0? await recordLedgerUsage(id,input.teamId,@@ -507,13 +584,19 @@ export async function retrieveProviders(input: {input.apiKeyId,credits,refundable,+ mcpCredits,): settled;- const charged =- settled && billable && credits > 0 && (recorded || !refundable);- if (charged && input.agentRequestId)+ const charged = billable && credits > 0 && (recorded || !refundable);+ if (charged && input.agentRequestId) {await recordAgentCredits(input.agentRequestId, id, credits);- if (!billable || recorded)+ await tallyAgentMcpCredits(+ { endpoint: "agent", agentRequestId: input.agentRequestId },+ `alexandria:${id}`,+ mcpCredits,+ );+ }+ if (!billable || (settled && recorded))void reportExchangeUsageBilling({requestId: id,status: billable ? "confirmed" : "void",@@ -570,6 +653,7 @@ async function recordLedgerUsage(apiKeyId: number | null,credits: number,refundable: boolean,+ mcpCredits: number,): Promise<boolean> {for (let attempt = 1; ; attempt++) {try {@@ -579,7 +663,11 @@ async function recordLedgerUsage(team_id: teamId,org_id: orgId,credits,- billing: { endpoint: "scrape", chargeId: id },+ billing: {+ endpoint: "scrape",+ chargeId: id,+ ...(mcpCredits > 0 ? { hostedMcp: true, mcpCredits } : {}),+ },is_extract: false,timestamp: new Date().toISOString(),originating_job_id: id,@@ -602,8 +690,8 @@ async function recordLedgerUsage();// refundable is only ever set with an org in hand; named again so the// type says so.- if (refundable && orgId !== null)- await autumnService.refundCredits({+ if (refundable && orgId !== null) {+ const refund = {teamId,orgId,value: credits,@@ -614,7 +702,12 @@ async function recordLedgerUsage(chargeId: id,},idempotencyKey: `fc:refund:scrape:${id}`,- });+ };+ // A split charge refunds each part to the balance it was settled on.+ await (mcpCredits > 0+ ? autumnService.refundHostedMcpCredits({ ...refund, mcpCredits })+ : autumnService.refundCredits(refund));+ }return false;}}diff --git a/apps/api/src/services/billing/__tests__/credit_billing.test.ts b/apps/api/src/services/billing/__tests__/credit_billing.test.tsindex a54e0b288..ddbd492f0 100644--- a/apps/api/src/services/billing/__tests__/credit_billing.test.ts+++ b/apps/api/src/services/billing/__tests__/credit_billing.test.ts@@ -20,6 +20,13 @@ const {vi.fn<(args: any) => Promise<{ tracked: boolean; mcpCredits: number }>>(),refundHostedMcpCredits: vi.fn<(args: any) => Promise<boolean>>(),}));+const { recordRequestCredits } = vi.hoisted(() => ({+ recordRequestCredits: vi.fn<(args: any) => Promise<boolean>>(),+}));++vi.mock("../../../lib/request-credits-store", () => ({+ recordRequestCredits,+}));vi.mock("../../../lib/withAuth", () => ({withAuth,@@ -65,6 +72,7 @@ beforeEach(() => {refundCredits.mockResolvedValue(undefined);isRoutedThroughFirebill.mockResolvedValue(false);trackHostedMcpCredits.mockResolvedValue({ tracked: true, mcpCredits: 0 });+ recordRequestCredits.mockResolvedValue(true);});describe("billTeam", () => {@@ -376,3 +384,55 @@ describe("billTeam MCP credits", () => {expect(trackCredits).not.toHaveBeenCalled();});});++describe("billTeam MCP credits on an agent run", () => {+ const agentCall = {+ endpoint: "search" as const,+ jobId: "job-1",+ chargeId: "charge-1",+ hostedMcp: true,+ agentRequestId: "agent-run",+ };++ it("adds the MCP credits part to the run's MCP tally under the charge", async () => {+ trackHostedMcpCredits.mockResolvedValue({ tracked: true, mcpCredits: 2 });++ await billTeam("team-1", "org-1", 3, 123, agentCall);++ expect(recordRequestCredits).toHaveBeenCalledExactlyOnceWith({+ requestId: "agent-run:mcp",+ jobId: "charge-1",+ credits: 2,+ });+ });++ it("tallies nothing when MCP credits paid nothing or the charge was refunded", async () => {+ await billTeam("team-1", "org-1", 3, 123, agentCall);++ trackHostedMcpCredits.mockResolvedValue({ tracked: true, mcpCredits: 2 });+ queueBillingOperation.mockResolvedValueOnce({ success: false });+ await billTeam("team-1", "org-1", 3, 123, agentCall);++ expect(recordRequestCredits).not.toHaveBeenCalled();+ });++ it("tallies nothing for a call outside an agent run", async () => {+ trackHostedMcpCredits.mockResolvedValue({ tracked: true, mcpCredits: 2 });++ await billTeam("team-1", "org-1", 3, 123, {+ ...agentCall,+ agentRequestId: undefined,+ });++ expect(recordRequestCredits).not.toHaveBeenCalled();+ });++ it("still bills when the tally cannot be written", async () => {+ trackHostedMcpCredits.mockResolvedValue({ tracked: true, mcpCredits: 2 });+ recordRequestCredits.mockRejectedValueOnce(new Error("store down"));++ await expect(+ billTeam("team-1", "org-1", 3, 123, agentCall),+ ).resolves.toMatchObject({ success: true });+ });+});diff --git a/apps/api/src/services/billing/agent-mcp-credits.ts b/apps/api/src/services/billing/agent-mcp-credits.tsnew file mode 100644index 000000000..5fa743ab3--- /dev/null+++ b/apps/api/src/services/billing/agent-mcp-credits.ts@@ -0,0 +1,36 @@+import { logger as rootLogger } from "../../lib/logger";+import { recordRequestCredits } from "../../lib/request-credits-store";+import type { BillingMetadata } from "./types";++/**+ * The request id a hosted MCP agent run tallies its calls' MCP credits under,+ * next to the run's own call-credits tally (see agentController).+ */+export function agentMcpCreditsRequestId(agentRequestId: string): string {+ return `${agentRequestId}:mcp`;+}++/**+ * Adds the MCP credits part of an agent call's charge to its run's tally.+ * `chargeKey` names the charge, so a retried charge counts once. Nothing is+ * recorded for a run that started no tally (any run not from the hosted MCP).+ */+export async function tallyAgentMcpCredits(+ billing: BillingMetadata,+ chargeKey: string,+ mcpCredits: number,+): Promise<void> {+ if (!billing.agentRequestId || !(mcpCredits > 0)) return;+ await recordRequestCredits({+ requestId: agentMcpCreditsRequestId(billing.agentRequestId),+ jobId: chargeKey,+ credits: mcpCredits,+ }).catch(error =>+ rootLogger.error("Failed to tally an agent call's MCP credits", {+ agentRequestId: billing.agentRequestId,+ chargeKey,+ mcpCredits,+ error,+ }),+ );+}diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex 6aefa9c10..c20be4271 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -6,6 +6,8 @@ import {} from "../autumn/autumn.service";import { toAutumnBillingProperties, type BillingMetadata } from "./types";import type { Logger } from "winston";+import { randomUUID } from "node:crypto";+import { tallyAgentMcpCredits } from "./agent-mcp-credits";/*** `org_id` is the team's Autumn customer. It is nullable here and nowhere@@ -84,6 +86,7 @@ export async function billTeam(// A track only happens with an org in hand; named again so the type says so.// A charge the MCP credits paid part of is refundable for that part even// when the rest was not tracked.+ let refunded = false;if (!result.success &&(trackedInRequest || mcpCredits > 0) &&@@ -118,9 +121,18 @@ export async function billTeam(await (mcpCredits > 0? autumnService.refundHostedMcpCredits({ ...refund, mcpCredits }): autumnService.refundCredits(refund));+ refunded = true;}}+ if (!refunded) {+ await tallyAgentMcpCredits(+ billing,+ billing.chargeId ?? randomUUID(),+ mcpCredits,+ );+ }+return result;},{ success: true, message: "No DB, bypassed." },diff --git a/apps/api/src/services/billing/types.ts b/apps/api/src/services/billing/types.tsindex 6c63d9732..493da4bae 100644--- a/apps/api/src/services/billing/types.ts+++ b/apps/api/src/services/billing/types.ts@@ -46,6 +46,13 @@ export type BillingMetadata = {* rest of this metadata, so a crawl's pages carry it too.*/hostedMcp?: boolean;+ /**+ * The agent run this charge belongs to, when the agent made the call. The+ * MCP credits part of a hosted MCP run's charge is tallied under it (see+ * tallyAgentMcpCredits), so a failed run can give that part back to MCP+ * credits rather than to regular credits.+ */+ agentRequestId?: string;/*** Credits of this charge the team's MCP credits paid, set by the charge that* tracks them to Autumn (billTeam or billScrapeJob). The ledger records them as their own row, tagged apart fromdiff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 80ba0c14e..a0885dafe 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -73,6 +73,7 @@ import {} from "../../lib/scrape-billing";import { getCrawlScope } from "../../lib/crawl-scope";import { billTeam } from "../billing/credit_billing";+import { tallyAgentMcpCredits } from "../billing/agent-mcp-credits";import { getBillingQueue } from "../queue-service";import type { Logger } from "winston";import {@@ -300,6 +301,7 @@ async function billScrapeJob(}}+ await tallyAgentMcpCredits(billing, job.id, mcpCredits);return creditsToBeBilled;} catch (error) {logger.error(@@ -329,6 +331,7 @@ async function billScrapeJob(billing,},);+ await tallyAgentMcpCredits(billing, job.id, mcpCredits);} else if (orgId) {const refund = {teamId: job.data.team_id,