Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
perf(alexandria): fewer round trips around a provider call (#5662)
Kris Oeidf271b3d76e9
diff --git a/apps/api/src/services/alexandria/access.ts b/apps/api/src/services/alexandria/access.tsindex d72e50300..0e363f072 100644--- a/apps/api/src/services/alexandria/access.ts+++ b/apps/api/src/services/alexandria/access.ts@@ -42,13 +42,15 @@ export async function authorizeProviders(orgId: string | null = null,clientIp?: string,billingCountry?: string,+ timeoutMs = 10000,): Promise<ExchangeResponse | undefined> {const providers = [...new Set(calls.map(call => call.provider))];+ const deadline = Date.now() + timeoutMs;const response = await exchangeRequest({teamId,path: "/v1/provider-terms/requirements",body: { providers },- timeoutMs: 10000,+ timeoutMs,clientIp,billingCountry,retryConnection: true,@@ -75,6 +77,7 @@ export async function authorizeProviders(return refusal(503,"Provider agreements are unavailable. No provider was executed.",+ { code: "precheck_unavailable", retryable: true },);if (config.USE_DB_AUTHENTICATION !== true) return undefined;@@ -92,10 +95,15 @@ export async function authorizeProviders(item.terms &&orgId !== null) {- ledger ??= await acceptedProviders(teamId, orgId, undefined, {- clientIp,- billingCountry,- });+ ledger ??= await acceptedProviders(+ teamId,+ orgId,+ Math.max(1, deadline - Date.now()),+ {+ clientIp,+ billingCountry,+ },+ );const accepted = ledger.get(item.provider);if (matchesAcceptance(accepted, item.terms) &&@@ -119,10 +127,15 @@ export async function authorizeProviders()continue;if (orgId !== null) {- ledger ??= await acceptedProviders(teamId, orgId, undefined, {- clientIp,- billingCountry,- });+ ledger ??= await acceptedProviders(+ teamId,+ orgId,+ Math.max(1, deadline - Date.now()),+ {+ clientIp,+ billingCountry,+ },+ );const accepted = ledger.get(item.provider);if (matchesAcceptance(accepted, item.terms)) continue;}diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex 59206ff20..4efd75bfc 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -85,6 +85,9 @@ const hash = (value: unknown) =>export const exchangeRequestId = (teamId: string, requestId: string) =>hash([teamId, requestId]);+const isCatalogueRead = (call: ProviderCall) =>+ call.provider === "firecrawl" && call.capability === "find-tools";+const unresolved = (chargeId: string) =>refusal(503,@@ -102,6 +105,22 @@ const relay = (response: ExchangeResponse): ExchangeResponse => {);};+/** Whether an Exchange answer accounts for every call of the request within its budget. */+const receiptMatches = (+ answer: z.infer<typeof answerSchema>,+ calls: ProviderCall[],+ maximumCredits: number,+) =>+ answer.results.length === calls.length &&+ answer.creditsCost <= maximumCredits &&+ answer.results.reduce((sum, item) => sum + (item.creditsCost ?? 0), 0) ===+ answer.creditsCost &&+ answer.results.every(+ (item, i) =>+ (item.provider ?? calls[i].provider) === calls[i].provider &&+ (item.capability ?? calls[i].capability) === calls[i].capability,+ );+export async function retrieveProviders(input: {teamId: string;/** The team's org β the Autumn customer this bills against. Null when none@@ -194,6 +213,7 @@ export async function retrieveProviders(input: {const key = `alexandria:retrieve:${id}`;const zeroDataRetention = input.zeroDataRetention === true;const usOnly = isUsOnly(input.flags);+ const deadline = Date.now() + input.timeoutMs;// Appended only when set so existing fingerprints stay stable.const fingerprint = hash([...(termsOnly@@ -205,7 +225,7 @@ export async function retrieveProviders(input: {const record: Retrieval = {fingerprint,phase: "executing",- deadline: Date.now() + input.timeoutMs,+ deadline,scrapeId: input.scrapeId,};const write = (next: Retrieval) =>@@ -267,6 +287,93 @@ export async function retrieveProviders(input: {return notExecuted(unresolved(id));}+ // A catalogue read is free and idempotent, so once its request id is claimed it skips the+ // agreement and quote lookups and the credit hold and goes to Exchange with a zero budget.+ // Exchange quotes against that budget itself and refuses with credit_budget_exceeded should+ // the read ever carry a price, in which case the full path below runs under the same claim+ // and deadline. The answer is recorded like any other, so a repeat of the id replays it.+ const firecrawlAccess = input.flags?.organizationDataSourceAccess?.firecrawl;+ if (+ input.calls.every(isCatalogueRead) &&+ (!firecrawlAccess || firecrawlAccess.status === "enabled")+ ) {+ const read = await exchangeRequest({+ teamId: input.teamId,+ path: "/v1/retrieve",+ body: { requests: input.calls },+ timeoutMs: input.timeoutMs,+ requestId: id,+ clientIp: input.clientIp,+ billingCountry: input.billingCountry,+ connectors: input.flags?.exchangeRetrieve === true,+ ...(input.resultAuthorization+ ? { resultAuthorization: input.resultAuthorization }+ : {}),+ ...(zeroDataRetention ? { zeroDataRetention } : {}),+ ...(usOnly ? { usOnly } : {}),+ maximumCredits: 0,+ retryConnection: true,+ }).catch(error => {+ logger.warn("Catalogue read unavailable", { error, chargeId: id });+ return undefined;+ });+ // A read that did not complete releases the id, so the same request can be sent again.+ const release = async (response: ExchangeResponse) => {+ await redisRateLimitClient.del(key);+ return { ...response, executed: true, scrapeId: input.scrapeId };+ };+ if (!read || read.status >= 500)+ return release(+ refusal(+ 503,+ "The catalogue read did not complete. Nothing was charged; resend it with the same x-request-id.",+ { code: "provider_unavailable", retryable: true },+ ),+ );+ const parsed =+ read.status === 200 ? answerSchema.safeParse(read.body) : undefined;+ if (parsed?.success && receiptMatches(parsed.data, input.calls, 0)) {+ const done: ExchangeResponse = { status: 200, body: parsed.data };+ void reportExchangeUsageBilling({+ requestId: id,+ status: billable ? "confirmed" : "void",+ billingReference: `alexandria:${id}`,+ });+ // The answer is delivered even when it cannot be recorded; the id is released instead,+ // so a repeat runs the read again rather than finding it stranded.+ await write(+ zeroDataRetention+ ? { ...record, phase: "done", responseWithheld: true }+ : { ...record, phase: "done", response: done },+ ).catch(async error => {+ logger.warn("Catalogue read answer was not recorded", {+ error,+ chargeId: id,+ });+ await redisRateLimitClient.del(key).catch(() => {});+ });+ return { ...done, executed: true, scrapeId: input.scrapeId };+ }+ const code = (read.body as { code?: unknown } | null)?.code;+ if (read.status >= 400 && code !== "credit_budget_exceeded")+ return release(relay(read));+ if (read.status < 400) {+ logger.error(+ "Catalogue read answered with an unexpected status or body",+ {+ chargeId: id,+ status: read.status,+ },+ );+ return release(+ refusal(+ 502,+ "The catalogue read returned a malformed answer. Nothing was charged; resend it with the same x-request-id.",+ ),+ );+ }+ }+const teamFeatureId = featureIdForBillingEndpoint("scrape");let featureId = input.hostedMcp? await autumnService.hostedMcpFeatureId(input.teamId, teamFeatureId)@@ -325,6 +432,7 @@ export async function retrieveProviders(input: {input.orgId,input.clientIp,input.billingCountry,+ Math.min(10000, remaining()),);if (denied) return refuse(denied);@@ -389,6 +497,7 @@ export async function retrieveProviders(input: {refusal(503,"Provider quote is unavailable. No provider was executed.",+ { code: "precheck_unavailable", retryable: true },),);if (quote.status !== 200) return refuse(relay(quote));@@ -445,7 +554,7 @@ export async function retrieveProviders(input: {? refusal(503,"Credit reservation is unavailable. No provider was executed.",- { code: "billing_unavailable" },+ { code: "billing_unavailable", retryable: true },): refusal(402,@@ -458,6 +567,9 @@ export async function retrieveProviders(input: {refusal(503,"Credit reservation is unavailable. No provider was executed.",+ hold.reason === "unavailable"+ ? { code: "billing_unavailable", retryable: true }+ : {},),);lockId = hold.lockId;@@ -527,19 +639,7 @@ export async function retrieveProviders(input: {const parsed = answerSchema.safeParse(response.body);if (!parsed.success) return fail("Alexandria answer was malformed");const answer = parsed.data;- const receiptMatches =- answer.results.length === input.calls.length &&- answer.creditsCost <= maximumCredits &&- answer.results.reduce((sum, item) => sum + (item.creditsCost ?? 0), 0) ===- answer.creditsCost &&- answer.results.every(- (item, i) =>- (item.provider ?? input.calls[i].provider) ===- input.calls[i].provider &&- (item.capability ?? input.calls[i].capability) ===- input.calls[i].capability,- );- if (!receiptMatches)+ if (!receiptMatches(answer, input.calls, maximumCredits))return fail("Alexandria receipt did not match the request");const credits = answer.creditsCost;diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.tsindex fa1a8d8cb..52c7eb809 100644--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts@@ -1274,7 +1274,7 @@ describe("firebill requests", () => {lockId: "lock-1",});- expect(result).toEqual({ status: "skipped" });+ expect(result).toEqual({ status: "skipped", reason: "unavailable" });expect(mockCheck).not.toHaveBeenCalled();});@@ -1290,7 +1290,7 @@ describe("firebill requests", () => {value: 10,lockId: "lock-1",}),- ).toEqual({ status: "skipped" });+ ).toEqual({ status: "skipped", reason: "unavailable" });mockFetch.mockRejectedValueOnce(new Error("ECONNREFUSED"));expect(@@ -1300,7 +1300,7 @@ describe("firebill requests", () => {value: 10,lockId: "lock-1",}),- ).toEqual({ status: "skipped" });+ ).toEqual({ status: "skipped", reason: "unavailable" });// A firebill-side hold may exist under this lock id; a direct-Autumn// retry of the same lock must never be attempted.@@ -1507,7 +1507,7 @@ describe("firebill requests", () => {value: 10,lockId: "lock-1",}),- ).toEqual({ status: "skipped" });+ ).toEqual({ status: "skipped", reason: "unavailable" });});it("hands the run token back on the finalize as the operation id", async () => {diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex ca6f64e62..5f96d1350 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -328,7 +328,7 @@ export class AutumnService {const unreachable = (): LockCreditsResult =>partnerJobToken? { status: "denied", reason: "gate_unavailable" }- : { status: "skipped" };+ : { status: "skipped", reason: "unavailable" };try {// The hold still lives in Autumn (firebill keeps no lock state), butdiff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.tsindex 961a724c5..d6871acdf 100644--- a/apps/api/src/services/autumn/types.ts+++ b/apps/api/src/services/autumn/types.ts@@ -52,14 +52,15 @@ export type LockDeniedReason =* Outcome of an Autumn credit lock attempt.** - `denied`: Autumn refused, or a partner's gate did β see `reason`.- * - `skipped`: billing not in effect; proceed without a lock.+ * - `skipped`: billing not in effect, or firebill gave no answer (`reason:+ * "unavailable"`); proceed without a lock.* - `locked`: reserved; finalize with `lockId`. `operationToken` is the* partner's id for this occurrence β hand it back on the finalize.*/export type LockCreditsResult =| { status: "locked"; lockId: string; operationToken?: string }| { status: "denied"; reason?: LockDeniedReason }- | { status: "skipped" };+ | { status: "skipped"; reason?: "unavailable" };export type FinalizeCreditsLockParams = {lockId: string;