Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(alexandria): refuse provider scrapes above an optional maxCredits (#5753)
Kris Oeie8f98a02ecaa
diff --git a/apps/api/openapi.json b/apps/api/openapi.jsonindex 8670db680..0fa47816a 100644--- a/apps/api/openapi.json+++ b/apps/api/openapi.json@@ -478,7 +478,13 @@}}],- "description": "Execute one or more catalogued provider tools instead of scraping a URL. Cannot be combined with `url`, `formats`, or other scrape options (400); the only allowed sibling keys are `timeout`, `origin`, and `integration`. Use the `x-request-id` request header as a client-chosen idempotency key (1 to 128 characters of letters, digits, `.`, `_`, `:`, `-`) — it is echoed back, and a completed request with the same key replays its original response and `scrape_id` instead of re-executing. To browse the catalogue and read tool contracts for free, call Find Tools: set `provider` to `firecrawl`, `capability` to `find-tools`, and `options` to a `FindToolsOptions` object. The result `data` is a `FindToolsData` page."+ "description": "Execute one or more catalogued provider tools instead of scraping a URL. Cannot be combined with `url`, `formats`, or other scrape options (400); the only allowed sibling keys are `timeout`, `origin`, `integration`, and `maxCredits`. Use the `x-request-id` request header as a client-chosen idempotency key (1 to 128 characters of letters, digits, `.`, `_`, `:`, `-`) — it is echoed back, and a completed request with the same key replays its original response and `scrape_id` instead of re-executing. To browse the catalogue and read tool contracts for free, call Find Tools: set `provider` to `firecrawl`, `capability` to `find-tools`, and `options` to a `FindToolsOptions` object. The result `data` is a `FindToolsData` page."+ },+ "maxCredits": {+ "type": "integer",+ "minimum": 0,+ "maximum": 9007199254740991,+ "description": "The most credits this request may use. Applies only to `alexandria` requests. The request is quoted first; when the quote's maximum is above `maxCredits`, it is refused with 402 `max_credits_exceeded` (the body includes `maximumCredits`) before any credits are held or any provider is executed. It caps only this request's quoted credits: requests a provider tool makes on its own that are billed separately, such as the child requests of an orchestrating tool, are not counted against it. A refused request frees its `x-request-id`, so it can be retried with the same key and a higher cap. A completed request replayed with the same `x-request-id` returns its original response; a different `maxCredits` on the replay is not applied."},"domainTools": {"type": "boolean",@@ -567,7 +573,7 @@}},"402": {- "description": "Payment required. Also returned as `insufficient_credits` when the team lacks the credits to execute an `alexandria` request; in that case nothing is executed.",+ "description": "Payment required. Also returned as `insufficient_credits` when the team lacks the credits to execute an `alexandria` request, and as `max_credits_exceeded` when an `alexandria` request's quote is above `maxCredits`; in both cases nothing is executed.","content": {"application/json": {"schema": {@@ -587,6 +593,10 @@},"chargeId": {"type": "string"+ },+ "maximumCredits": {+ "type": "integer",+ "description": "With `max_credits_exceeded`: the most credits the request was quoted to need."}}}diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex 0ea512fe6..fd65b17ed 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -76,6 +76,7 @@ const providerScrapeSchema = z.strictObject({origin: z.string().default("api"),integration: z.string().nullable().optional(),zeroDataRetention: z.boolean().optional(),+ maxCredits: z.number().int().nonnegative().safe().optional(),__agentInterop: agentInteropSchema.optional(),});@@ -189,6 +190,7 @@ export async function providerScrapeController(clientIp: req.ip,billingCountry: await billingCountryForTeam(req.auth.team_id),rateLimitMultiplier: req.acuc.rate_limit_multiplier,+ ...(body.maxCredits === undefined ? {} : { maxCredits: body.maxCredits }),});} catch (error) {logger.error("Provider scrape unavailable", {diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex 4efd75bfc..be8b69cff 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -144,6 +144,8 @@ export async function retrieveProviders(input: {billingCountry?: string;/** The team's Autumn rate-limit multiplier, which sizes its index budgets. */rateLimitMultiplier: number;+ /** The most the caller allows: a quote above it is refused before anything is held or run. */+ maxCredits?: number;}): Promise<ProviderRetrieval> {const notExecuted = (response: ExchangeResponse): ProviderRetrieval => ({...response,@@ -509,6 +511,15 @@ export async function retrieveProviders(input: {refusal(502, "Provider quote was malformed. No provider was executed."),);maximumCredits = parsedQuote.data.maximumCredits;+ // Execution is bound to this quote (Exchange refuses above it), so the caller's cap binds too.+ if (input.maxCredits !== undefined && maximumCredits > input.maxCredits)+ return refuse(+ refusal(+ 402,+ `This request needs up to ${maximumCredits} credits, above maxCredits ${input.maxCredits}. No provider was executed.`,+ { code: "max_credits_exceeded", maximumCredits },+ ),+ );if (billable && maximumCredits > 0) {if (!config.USE_DB_AUTHENTICATION)