Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): handle large context payloads in alexandria (#4701)
Developers Digeste3e324e3d425
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex 860891f5f..0f3e6fd2d 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -127,6 +127,7 @@ export async function providerScrapeController(apiKeyId: req.acuc.api_key_id ?? null,flags: req.acuc.flags,calls: body.alexandria,+ resultAuthorization: req.get("authorization"),requestId,scrapeId: uuidv7(),timeoutMs: body.timeout,diff --git a/apps/api/src/services/alexandria/README.md b/apps/api/src/services/alexandria/README.mdindex 37d2a64f2..ae5d257ba 100644--- a/apps/api/src/services/alexandria/README.md+++ b/apps/api/src/services/alexandria/README.md@@ -79,3 +79,9 @@ API, Redis, Exchange (`FIRE_EXCHANGE_URL`; a non-blank`EXCHANGE_INTERNAL_SECRET` enables the usage report) and Autumn for paidexecution. Paid settlement is covered by mocked tests only and has not runagainst an Autumn sandbox.++Bash source loading (`firecrawl/bash` with `options.requestId`) must be sent as a+single-call request. Mixed batches are rejected before billing or dispatch because+caller authorization is request-scoped. Workspace reuse (`options.workspaceId`)+does not forward the caller credential. Exchange consumes the forwarded credential+only in its Bash saved-scrape reader; provider integrations use their own credentials.diff --git a/apps/api/src/services/alexandria/client.ts b/apps/api/src/services/alexandria/client.tsindex b4e0beca0..272854f33 100644--- a/apps/api/src/services/alexandria/client.ts+++ b/apps/api/src/services/alexandria/client.ts@@ -15,6 +15,7 @@ export async function exchangeRequest(input: {timeoutMs: number;requestId?: string;maximumCredits?: number;+ resultAuthorization?: string;}): Promise<ExchangeResponse> {if (!config.FIRE_EXCHANGE_URL) throw new Error("Exchange is not configured");const base = config.FIRE_EXCHANGE_URL.replace(/\/+$/, "");@@ -25,6 +26,9 @@ export async function exchangeRequest(input: {signal: AbortSignal.timeout(Math.max(1, input.timeoutMs)),headers: {"content-type": "application/json",+ ...(input.resultAuthorization+ ? { authorization: input.resultAuthorization }+ : {}),"x-exchange-team-id": input.teamId,"x-exchange-extended-catalog-access": "true",...(input.requestId ? { "x-request-id": input.requestId } : {}),diff --git a/apps/api/src/services/alexandria/retrieve.test.ts b/apps/api/src/services/alexandria/retrieve.test.tsindex 20457f3df..7e4858773 100644--- a/apps/api/src/services/alexandria/retrieve.test.ts+++ b/apps/api/src/services/alexandria/retrieve.test.ts@@ -278,3 +278,97 @@ it("treats a team with no org as a skipped hold and executes nothing", async ()expect(executions()).toHaveLength(0);expect(mocks.store.size).toBe(0);});++it.each([false, true])(+ "rejects mixed Bash source loading before side effects (Bash first: %s)",+ async bashFirst => {+ const bash = {+ provider: "firecrawl",+ capability: "bash",+ options: { requestId: "source", command: "ls" },+ };+ const result = await run({+ calls: bashFirst ? [bash, call] : [call, bash],+ resultAuthorization: "Bearer caller",+ });+ expect(result).toMatchObject({ status: 400, executed: false });+ expect(mocks.request).not.toHaveBeenCalled();+ expect(mocks.authorize).not.toHaveBeenCalled();+ expect(mocks.lock).not.toHaveBeenCalled();+ expect(mocks.store.size).toBe(0);+ },+);++it("allows standalone Bash source loading and forwards credentials only to execution", async () => {+ const bash = {+ provider: "firecrawl",+ capability: "bash",+ options: { requestId: "source", command: "ls" },+ };+ exchangeAnswers({+ success: true,+ creditsCost: 0,+ results: [{ ...bash, creditsCost: 0, data: { workspaceId: "workspace" } }],+ });+ const result = await run({+ calls: [bash],+ resultAuthorization: "Bearer caller",+ });+ expect(result).toMatchObject({ status: 200, executed: true });+ expect(executions()).toHaveLength(1);+ expect(executions()[0][0].resultAuthorization).toBe("Bearer caller");+ expect(+ mocks.request.mock.calls+ .filter(([arg]) => arg.path !== "/v1/retrieve")+ .every(([arg]) => arg.resultAuthorization === undefined),+ ).toBe(true);+ expect([...mocks.store.values()].join("")).not.toContain("Bearer caller");+});++it.each([+ {+ provider: "firecrawl",+ capability: "bash",+ options: { workspaceId: "workspace", command: "ls" },+ },+ {+ provider: "firecrawl",+ capability: "find-tools",+ options: { requestId: "source" },+ },+ { provider: "other", capability: "bash", options: { requestId: "source" } },+])("does not forward credentials for other calls: %j", async other => {+ exchangeAnswers({+ success: true,+ creditsCost: 0,+ results: [call, other].map(entry => ({+ ...entry,+ creditsCost: 0,+ data: {},+ })),+ });+ expect(+ await run({ calls: [call, other], resultAuthorization: "Bearer caller" }),+ ).toMatchObject({ status: 200, executed: true });+ expect(executions()[0][0].resultAuthorization).toBeUndefined();+});++it.each([123, null, false, {}, []])(+ "never forwards credentials for a non-string Bash requestId: %j",+ async requestId => {+ const bash = {+ provider: "firecrawl",+ capability: "bash",+ options: { requestId, command: "ls" },+ };+ exchangeAnswers(+ { success: false, error: "invalid option", code: "invalid_option" },+ 400,+ );+ expect(+ await run({ calls: [call, bash], resultAuthorization: "Bearer caller" }),+ ).toMatchObject({ status: 400 });+ expect(executions()).toHaveLength(1);+ expect(executions()[0][0].resultAuthorization).toBeUndefined();+ },+);diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex 390d206ab..0e911f391 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -88,6 +88,7 @@ export async function retrieveProviders(input: {scrapeId: string;timeoutMs: number;bypassBilling?: boolean;+ resultAuthorization?: string;}): Promise<ProviderRetrieval> {const notExecuted = (response: ExchangeResponse): ProviderRetrieval => ({...response,@@ -104,6 +105,20 @@ export async function retrieveProviders(input: {if (Buffer.byteLength(JSON.stringify(input.calls)) > 256 * 1024)return notExecuted(refusal(400, "Provider options exceed 256 KB."));+ const loadsBashResult = input.calls.some(+ call =>+ call.provider === "firecrawl" &&+ call.capability === "bash" &&+ typeof call.options?.requestId === "string",+ );+ if (loadsBashResult && input.calls.length !== 1)+ return notExecuted(+ refusal(+ 400,+ "Bash source loading must be sent as a separate request; do not batch it with other calls.",+ ),+ );+const billable = !input.bypassBilling;const id = hash([input.teamId, input.requestId]);const key = `alexandria:retrieve:${id}`;@@ -315,6 +330,9 @@ export async function retrieveProviders(input: {body: { requests: input.calls },timeoutMs: remaining(),requestId: id,+ ...(loadsBashResult && input.resultAuthorization+ ? { resultAuthorization: input.resultAuthorization }+ : {}),maximumCredits,}).catch(error => {throw new Error(`Exchange did not answer: ${error?.message ?? error}`);