Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): provider discovery through Search and billed execution through Scrape (#4621)
Developers Digestcc06662bdc1b
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex fb1e655ec..8b130f4ae 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -301,6 +301,7 @@ const configSchema = z.object({// Exchange (routed data sources service)FIRE_EXCHANGE_URL: z.url().optional(),+ EXCHANGE_INTERNAL_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),// Fire EngineFIRE_ENGINE_BETA_URL: z.string().optional(),diff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.tsnew file mode 100644index 000000000..54ec54d0f--- /dev/null+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts@@ -0,0 +1,137 @@+import express from "express";+import request from "supertest";+const mocks = vi.hoisted(() => ({ retrieve: vi.fn(), log: vi.fn() }));+vi.mock("../../services/alexandria/retrieve", () => ({+ REQUEST_ID_PATTERN: /^[A-Za-z0-9._:-]{1,128}$/,+ retrieveProviders: mocks.retrieve,+}));+vi.mock("../../services/logging/log_job", () => ({ logRequest: mocks.log }));+vi.mock("../../lib/key-restriction", () => ({+ checkKeyFormatRestriction: async () => ({ allowed: true }),+}));+vi.mock("../../lib/agent-interop", () => ({+ isAgentInteropSecretValid: (value: string) => value === "test-secret",+}));+vi.mock("../../config", () => ({+ config: {+ FIRE_EXCHANGE_URL: "https://x",+ AGENT_INTEROP_SECRET: "test-secret",+ },+}));+import { providerScrapeController } from "./scrape-alexandria";++const call = {+ provider: "fred",+ capability: "categories/category",+ options: {},+};+const app = express();+app.use(express.json());+app.use((req, _res, next) => {+ Object.assign(req, {+ auth: { team_id: "team" },+ acuc: { api_key_id: 12, flags: { exchangeRetrieve: true } },+ });+ next();+});+app.post("/v2/scrape", (req, res) => providerScrapeController(req as any, res));+app.post("/exchange/retrieve", (req, res) =>+ providerScrapeController(req as any, res, true),+);+const result = (results: unknown[], executed = true) => ({+ status: 200,+ body: { success: true, creditsCost: 0, results },+ executed,+ scrapeId: "scrape-1",+});++beforeEach(() => {+ vi.clearAllMocks();+ mocks.log.mockResolvedValue(undefined);+ mocks.retrieve.mockResolvedValue(+ result([{ ...call, creditsCost: 0, data: {} }]),+ );+});++it("returns the Scrape contract, shares identity with the legacy route, and logs once per execution", async () => {+ const response = await request(app)+ .post("/v2/scrape")+ .set("x-request-id", "same-request")+ .send({ alexandria: call });+ expect(response.status).toBe(200);+ expect(response.body).toEqual({+ success: true,+ scrape_id: "scrape-1",+ data: { alexandria: [expect.any(Object)], creditsCost: 0 },+ });+ expect(response.headers["x-request-id"]).toBe("same-request");+ expect(mocks.retrieve).toHaveBeenCalledWith(+ expect.objectContaining({+ calls: [call],+ requestId: "same-request",+ apiKeyId: 12,+ }),+ );++ mocks.retrieve.mockResolvedValue(+ result([{ ...call, creditsCost: 0, data: {} }], false),+ );+ await request(app)+ .post("/exchange/retrieve")+ .set("x-request-id", "same-request")+ .send(call);+ const [first, second] = mocks.retrieve.mock.calls.map(([arg]) => arg);+ expect({ ...first, scrapeId: undefined }).toEqual({+ ...second,+ scrapeId: undefined,+ });+ expect(mocks.log).toHaveBeenCalledTimes(1);+});++it("relays a failed single legacy call as an error, not a success", async () => {+ mocks.retrieve.mockResolvedValue(+ result([+ {+ ...call,+ creditsCost: 0,+ error: { code: "credential_missing", message: "No key.", status: 503 },+ },+ ]),+ );+ const response = await request(app).post("/exchange/retrieve").send(call);+ expect(response.status).toBe(503);+ expect(response.body).toEqual({+ success: false,+ code: "credential_missing",+ error: "No key.",+ });+});++it("only lets trusted agent interop bypass billing, and prefers its request id", async () => {+ const untrusted = await request(app)+ .post("/v2/scrape")+ .send({+ alexandria: call,+ __agentInterop: { auth: "wrong", requestId: "a", shouldBill: false },+ });+ expect(untrusted.status).toBe(403);+ expect(mocks.retrieve).not.toHaveBeenCalled();++ const trusted = await request(app)+ .post("/v2/scrape")+ .set("x-request-id", "hop-id")+ .send({+ alexandria: call,+ __agentInterop: {+ auth: "test-secret",+ requestId: "agent-id",+ shouldBill: false,+ boostConcurrency: true,+ },+ });+ expect(trusted.status).toBe(200);+ expect(mocks.retrieve).toHaveBeenCalledWith(+ expect.objectContaining({ requestId: "agent-id", bypassBilling: true }),+ );+ expect(mocks.log).not.toHaveBeenCalled();+});diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsnew file mode 100644index 000000000..51f4e63b3--- /dev/null+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -0,0 +1,174 @@+import { z } from "zod";+import type { Response } from "express";+import { randomUUID } from "node:crypto";+import { config } from "../../config";+import { logger } from "../../lib/logger";+import { isAgentInteropSecretValid } from "../../lib/agent-interop";+import { externalRequestId } from "../../lib/external-request-id";+import { getScrapeZDR } from "../../lib/zdr-helpers";+import { checkKeyFormatRestriction } from "../../lib/key-restriction";+import { logRequest } from "../../services/logging/log_job";+import {+ callsSchema,+ callSchema,+ answerSchema,+} from "../../services/alexandria/contracts";+import {+ REQUEST_ID_PATTERN,+ retrieveProviders,+} from "../../services/alexandria/retrieve";+import type { RequestWithAuth } from "./types";++const providerScrapeSchema = z.strictObject({+ alexandria: z.preprocess(+ value => (Array.isArray(value) ? value : [value]),+ callsSchema,+ ),+ timeout: z+ .number()+ .int()+ .positive()+ .default(50000)+ .transform(value => Math.min(value, 50000)),+ origin: z.string().default("api"),+ integration: z.string().nullable().optional(),+ __agentInterop: z+ .strictObject({+ auth: z.string(),+ requestId: z.string(),+ shouldBill: z.boolean(),+ boostConcurrency: z.boolean().optional(),+ })+ .optional(),+});++export async function providerScrapeController(+ req: RequestWithAuth<any, any, any>,+ res: Response,+ legacy = false,+) {+ const legacyBody = legacy+ ? z+ .union([callSchema, z.strictObject({ requests: callsSchema })])+ .parse(req.body)+ : null;+ const body = providerScrapeSchema.parse(+ legacyBody+ ? {+ alexandria:+ "requests" in legacyBody ? legacyBody.requests : [legacyBody],+ }+ : req.body,+ );++ if (body.__agentInterop) {+ if (!config.AGENT_INTEROP_SECRET)+ return res+ .status(403)+ .json({ success: false, error: "Agent interop is not enabled." });+ if (!isAgentInteropSecretValid(body.__agentInterop.auth))+ return res+ .status(403)+ .json({ success: false, error: "Invalid agent interop." });+ }+ const requestId =+ body.__agentInterop?.requestId ?? req.get("x-request-id") ?? randomUUID();+ if (REQUEST_ID_PATTERN.test(requestId))+ res.setHeader("x-request-id", requestId);++ if (!req.acuc?.flags?.exchangeRetrieve)+ return res.status(403).json({+ success: false,+ error: "This endpoint is not enabled for this team.",+ });+ if (getScrapeZDR(req.acuc.flags) === "forced")+ return res.status(403).json({+ success: false,+ error: "Provider tools do not support zero data retention.",+ });+ if (+ (req as any).agentIndexOnly ||+ ["pending", "blocked"].includes(req.acuc._agentSponsor?.status ?? "")+ )+ return res.status(403).json({+ success: false,+ code: "AGENT_INDEX_ONLY",+ error: "Verify this API key before executing provider tools.",+ });+ const restriction = await checkKeyFormatRestriction(+ ["json"],+ [],+ req.acuc.api_key_id,+ req.acuc.flags,+ );+ if (!restriction.allowed)+ return res+ .status(restriction.status)+ .json({ success: false, error: restriction.error });+ if (!config.FIRE_EXCHANGE_URL)+ return res+ .status(503)+ .json({ success: false, error: "This endpoint is not available." });++ let result;+ try {+ result = await retrieveProviders({+ teamId: req.auth.team_id,+ apiKeyId: req.acuc.api_key_id ?? null,+ flags: req.acuc.flags,+ calls: body.alexandria,+ requestId,+ scrapeId: randomUUID(),+ timeoutMs: body.timeout,+ bypassBilling: body.__agentInterop?.shouldBill === false,+ });+ } catch (error) {+ logger.error("Provider scrape unavailable", {+ error,+ teamId: req.auth.team_id,+ });+ return res.status(503).json({+ success: false,+ error: "Provider request unavailable. Retry with the same x-request-id.",+ });+ }+ if (result.executed && !body.__agentInterop)+ void logRequest({+ id: result.scrapeId,+ kind: "scrape",+ api_version: "v2",+ external_request_id: externalRequestId(req),+ team_id: req.auth.team_id,+ api_key_id: req.acuc.api_key_id ?? null,+ origin: body.origin,+ integration: body.integration ?? null,+ target_hint: `alexandria:${body.alexandria.map(call => `${call.provider}/${call.capability}`).join(",")}`,+ zeroDataRetention: false,+ }).catch(error =>+ logger.warn("Provider request logging failed", {+ error,+ scrapeId: result.scrapeId,+ }),+ );++ if (result.status !== 200) return res.status(result.status).json(result.body);+ const answer = answerSchema.parse(result.body);+ if (legacy) {+ if (legacyBody && "requests" in legacyBody) return res.json(answer);+ const first = answer.results[0];+ if (first.error) {+ const status = first.error.status ?? 502;+ return res.status(status >= 400 && status <= 599 ? status : 502).json({+ success: false,+ code: first.error.code,+ error: first.error.message,+ });+ }+ return res.json({ success: true, ...first });+ }+ return res.json({+ success: true,+ scrape_id: result.scrapeId,+ data: { alexandria: answer.results, creditsCost: answer.creditsCost },+ });+}diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex da7e24665..74940dd1a 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -1,4 +1,6 @@import { Response } from "express";+import { providerScrapeController } from "./scrape-alexandria";+import { discoverTools } from "../../search/alexandria";import { config } from "../../config";import { logger as _logger } from "../../lib/logger";import {@@ -50,6 +52,8 @@ export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,res: Response<ScrapeResponse>,) {+ if (req.body && "alexandria" in req.body)+ return providerScrapeController(req, res);// Resolved before the root span starts so the whole request trace stays// unrecorded for zero-data-retention requests (see otel-tracer).const zeroDataRetentionTrace =@@ -152,6 +156,16 @@ export async function scrapeController(getScrapeZDR(req.acuc?.flags) === "forced" ||(req.body.zeroDataRetention ?? false) ||(req.body.lockdown ?? false);+ if (+ req.body.domainTools &&+ (!req.acuc?.flags?.exchangeRetrieve || zeroDataRetention)+ )+ return res.status(403).json({+ success: false,+ error: !req.acuc?.flags?.exchangeRetrieve+ ? "The alexandria source is not enabled for this team."+ : "Provider discovery requires access and does not support zero data retention.",+ });const billing: BillingMetadata = req.body.__agentInterop? { endpoint: "agent" as const, jobId }: { endpoint: "scrape" as const, jobId };@@ -625,6 +639,29 @@ export async function scrapeController(concurrencyLimited,concurrencyQueueDurationMs: lockTime || undefined,});+ const tools =+ req.body.domainTools && config.FIRE_EXCHANGE_URL+ ? await discoverTools(+ {+ teamId: req.auth.team_id,+ urls: [+ ...new Set(+ [+ req.body.url,+ doc!.metadata?.sourceURL,+ doc!.metadata?.url,+ ].filter((u): u is string => typeof u === "string"),+ ),+ ],+ limit: 24,+ timeoutMs: 10000,+ },+ logger,+ ).catch(error => {+ logger.warn("Domain tool discovery failed", { error });+ return undefined;+ })+ : undefined;return res.status(200).json({success: true,@@ -637,7 +674,9 @@ export async function scrapeController(? lockTime || 0: undefined,},+ ...(tools ? { tools: tools.items } : {}),},+ ...(tools?.warning ? { warning: tools.warning } : {}),scrape_id: origin?.includes("website") ? jobId : undefined,});},diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 41d92d925..3296b4dcd 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -37,6 +37,7 @@ import {import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { resolveThreatProtection } from "../../lib/threat-protection/request";+import { isToolsOnlySearch } from "../../search/alexandria";import {actionTypesOf,checkKeyEndpointRestriction,@@ -129,6 +130,27 @@ async function searchControllerInner(typeof req.body?.origin === "string" ? req.body.origin : undefined;req.body = searchRequestSchema.parse(req.body);+ const wantsTools = req.body.sources.some(+ source => source.type === "alexandria",+ );+ if (wantsTools && !req.body.query.trim())+ return res.status(400).json({+ success: false,+ error: "A query is required for tool search.",+ });+ if (+ (wantsTools || req.body.domainTools) &&+ (!req.acuc?.flags?.exchangeRetrieve ||+ teamForcedKind ||+ req.body.enterprise?.some(mode => mode === "zdr" || mode === "anon"))+ )+ return res.status(403).json({+ success: false,+ error: !req.acuc?.flags?.exchangeRetrieve+ ? "The alexandria source is not enabled for this team."+ : "Provider discovery requires access and does not support zero data retention.",+ });+const requestedFormats = formatTypesOf(req.body.scrapeOptions?.formats);const keyRestriction = await checkKeyFormatRestriction(requestedFormats,@@ -252,8 +274,9 @@ async function searchControllerInner(});}+ const toolsOnly = isToolsOnlySearch(req.body.sources, req.body.categories);const projectedKeylessCredits =- !isSearchPreview && shouldBill+ !isSearchPreview && shouldBill && !toolsOnly? projectSearchTotalCredits({limit: req.body.limit,@@ -295,6 +318,7 @@ async function searchControllerInner(enterprise: req.body.enterprise,scrapeOptions: req.body.scrapeOptions,highlights: req.body.highlights,+ domainTools: req.body.domainTools,timeout: req.body.timeout,},{@@ -430,6 +454,7 @@ async function searchControllerInner(data: result.response,creditsUsed: result.totalCredits,id: jobId,+ ...(result.toolsWarning ? { warning: result.toolsWarning } : {}),});} catch (error) {if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 9b0dd1024..a1e9d421e 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1116,6 +1116,7 @@ const scrapeRequestSchemaBase = baseScrapeOptions.extend({origin: z.string().optional().prefault("api"),integration: integrationSchema.optional().transform(val => val || null),zeroDataRetention: z.boolean().optional(),+ domainTools: z.boolean().optional(),__agentInterop: z.object({auth: z.string(),@@ -1538,7 +1539,9 @@ export type ScrapeResponse =| {success: true;warning?: string;- data: Document;+ data: Document & {+ tools?: import("../../services/alexandria/contracts").DiscoveredTool[];+ };scrape_id?: string;};@@ -1888,6 +1891,7 @@ type Account = {};export type TeamFlags = {+ exchangeRetrieve?: boolean;ignoreRobots?: "disabled" | "allowed" | "forced";customRobotsAgent?: "disabled" | "allowed";threatProtection?: "disabled" | "allowed" | "forced";@@ -2323,13 +2327,16 @@ export const searchRequestSchema = zsources: z.union([// Array of strings (simple format)- z.array(z.enum(["web", "images", "news"])),+ z.array(z.enum(["web", "images", "news", "alexandria"])),// Array of objects (advanced format)z.array(z.union([webSearchSourceOptions,imagesSearchSourceOptions,newsSearchSourceOptions,+ z.strictObject({+ type: z.literal("alexandria"),+ }),]),),])@@ -2367,6 +2374,7 @@ export const searchRequestSchema = z// our index. When omitted, the caller integration and rollout cohort decide// whether generated highlights are returned or only run in shadow mode.highlights: z.boolean().optional(),+ domainTools: z.boolean().optional(),__searchPreviewToken: z.string().optional(),threatProtection: threatProtectionOverrideSchema.optional(),scrapeOptions: baseScrapeOptionsdiff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.tsindex 0d5dac8ee..440bf134f 100644--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@ -164,6 +164,7 @@ export interface WebSearchResult {export type SearchResultType = "web" | "images" | "news";export interface SearchV2Response {+ tools?: import("../services/alexandria/contracts").DiscoveredTool[];web?: WebSearchResult[];images?: ImageSearchResult[];news?: NewsSearchResult[];diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.tsindex 6a4f0a69d..cfdd098bc 100644--- a/apps/api/src/lib/exchange.test.ts+++ b/apps/api/src/lib/exchange.test.ts@@ -292,9 +292,7 @@ describe("Exchange routing", () => {it("accepts only formats the Exchange can return directly", () => {expect(isSupportedExchangeFormatRequest(undefined)).toBe(true);- expect(isSupportedExchangeFormatRequest([{ type: "markdown" }])).toBe(- true,- );+ expect(isSupportedExchangeFormatRequest([{ type: "markdown" }])).toBe(true);expect(isSupportedExchangeFormatRequest(["json"])).toBe(true);expect(isSupportedExchangeFormatRequest([@@ -305,9 +303,9 @@ describe("Exchange routing", () => {expect(isSupportedExchangeFormatRequest([{ type: "html" }])).toBe(false);// deterministicJson extractors run against page HTML, which Exchange// responses do not carry.- expect(isSupportedExchangeFormatRequest([{ type: "deterministicJson" }])).toBe(- false,- );+ expect(+ isSupportedExchangeFormatRequest([{ type: "deterministicJson" }]),+ ).toBe(false);expect(isSupportedExchangeFormatRequest([])).toBe(false);});@@ -403,15 +401,21 @@ describe("Exchange routing", () => {terms: ACME_TERMS,});- expect(getThirdPartyDataTermsRequiredResponse(ACME_TERMS)).toMatchObject({+ const response = getThirdPartyDataTermsRequiredResponse(ACME_TERMS);+ expect(response).toMatchObject({success: false,code: "THIRD_PARTY_DATA_TERMS_REQUIRED",requiresAction: {type: "accept_terms",terms: "acme",version: "2026-01-01",+ url: expect.stringMatching(/\/app\/alexandria\/acme$/),},});+ // The message carries the provider, version and link, since most clients relay only `error`.+ expect(response.error).toContain("acme");+ expect(response.error).toContain("2026-01-01");+ expect(response.error).toContain(response.requiresAction.url);});it("requires current terms when the accepted version is stale", async () => {@@ -562,7 +566,10 @@ describe("Exchange routing", () => {"https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing",expect.objectContaining({method: "POST",- body: JSON.stringify({ status: "confirmed", billingReference: "bill-1" }),+ body: JSON.stringify({+ status: "confirmed",+ billingReference: "bill-1",+ }),}),);diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.tsindex 41518418b..937ddd8ac 100644--- a/apps/api/src/lib/exchange.ts+++ b/apps/api/src/lib/exchange.ts@@ -70,8 +70,6 @@ type ExchangeProvider = {const SUPPORTED_FORMATS = new Set(["markdown", "json"]);const EXCHANGE_BETA_FLAG = "professionalProfileCompanyDataBeta";const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED";-const THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE =- "An organization admin must accept this data source's terms before this URL can be processed.";const EXCHANGE_PROVIDERS_PATH = "/v1/providers";const EXCHANGE_PROVIDERS_TIMEOUT_MS = 2_000;@@ -515,20 +513,22 @@ export async function canUseExchangeForRequest(return (await getExchangeAccessForRequest(input)).allowed;}-function getThirdPartyDataTermsSettingsUrl(): string {- return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/settings?tab=data-sources`;+// Terms are keyed by provider id, so the key doubles as the provider page to accept them on.+function getThirdPartyDataTermsUrl(terms: ExchangeTerms): string {+ return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/alexandria/${encodeURIComponent(terms.key)}`;}export function getThirdPartyDataTermsRequiredResponse(terms: ExchangeTerms) {+ const url = getThirdPartyDataTermsUrl(terms);return {success: false as const,code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE as "THIRD_PARTY_DATA_TERMS_REQUIRED",- error: THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE,+ error: `An organization admin must accept the ${terms.key} provider's terms (version ${terms.version}) before this request can run. Accept them at ${url}`,requiresAction: {type: "accept_terms",terms: terms.key,version: terms.version,- url: getThirdPartyDataTermsSettingsUrl(),+ url,},};}@@ -599,24 +599,72 @@ export async function reportExchangeBilling(input: {return false;}+ return deliverBillingReport({+ url: `${baseUrl}/v1/access-events/${encodeURIComponent(input.accessEventId)}/billing`,+ headers: { "Content-Type": "application/json" },+ body: {+ status: input.status,+ ...(input.billingReference === undefined+ ? {}+ : { billingReference: input.billingReference }),+ },+ retryNotFound: false,+ context: { accessEventId: input.accessEventId, status: input.status },+ });+}++/**+ * Report the billing outcome of a tool execution's usage rows, keyed by the+ * `x-request-id` sent with `/v1/retrieve`. Internal-secret route; usage is+ * recorded asynchronously on the Exchange, so a 404 is retried like a 5xx.+ */+export async function reportExchangeUsageBilling(input: {+ requestId: string;+ status: "confirmed" | "void";+ billingReference?: string;+}): Promise<boolean> {+ const baseUrl = getExchangeBaseUrl();+ if (!baseUrl || !config.EXCHANGE_INTERNAL_SECRET) {+ return false;+ }++ return deliverBillingReport({+ url: `${baseUrl}/v1/usage-events/billing`,+ headers: {+ "Content-Type": "application/json",+ "x-exchange-secret": config.EXCHANGE_INTERNAL_SECRET,+ },+ body: [+ {+ requestId: input.requestId,+ status: input.status,+ ...(input.billingReference === undefined+ ? {}+ : { billingReference: input.billingReference }),+ },+ ],+ retryNotFound: true,+ context: { requestId: input.requestId, status: input.status },+ });+}++async function deliverBillingReport(input: {+ url: string;+ headers: Record<string, string>;+ body: unknown;+ retryNotFound: boolean;+ context: Record<string, unknown>;+}): Promise<boolean> {for (let attempt = 1; attempt <= EXCHANGE_BILLING_ATTEMPTS; attempt++) {let retryAfterMs: number | undefined;try {- const response = await fetch(- `${baseUrl}/v1/access-events/${encodeURIComponent(input.accessEventId)}/billing`,- {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify({- status: input.status,- ...(input.billingReference === undefined- ? {}- : { billingReference: input.billingReference }),- }),- signal: AbortSignal.timeout(EXCHANGE_BILLING_TIMEOUT_MS),- },- );+ const response = await fetch(input.url, {+ method: "POST",+ headers: input.headers,+ body: JSON.stringify(input.body),+ signal: AbortSignal.timeout(EXCHANGE_BILLING_TIMEOUT_MS),+ });if (response.ok) {return true;@@ -624,11 +672,15 @@ export async function reportExchangeBilling(input: {// 4xx responses other than 429 are definitive (conflict, unknown// event) - the Exchange has spoken and a retry cannot change the- // answer. 429 is transient rate limiting and retries.- if (response.status < 500 && response.status !== 429) {+ // answer. 429 is transient rate limiting and retries, as does a 404+ // where the caller knows the rows are written asynchronously.+ if (+ response.status < 500 &&+ response.status !== 429 &&+ !(input.retryNotFound && response.status === 404)+ ) {rootLogger.warn("Exchange billing report rejected", {- accessEventId: input.accessEventId,- status: input.status,+ ...input.context,statusCode: response.status,});return false;@@ -639,15 +691,13 @@ export async function reportExchangeBilling(input: {}rootLogger.warn("Exchange billing report failed", {- accessEventId: input.accessEventId,- status: input.status,+ ...input.context,statusCode: response.status,attempt,});} catch (error) {rootLogger.warn("Exchange billing report errored", {- accessEventId: input.accessEventId,- status: input.status,+ ...input.context,attempt,error,});diff --git a/apps/api/src/routes/exchange.ts b/apps/api/src/routes/exchange.tsindex 31b31e988..cc5709fed 100644--- a/apps/api/src/routes/exchange.ts+++ b/apps/api/src/routes/exchange.ts@@ -1,4 +1,5 @@import { bountyBlocklistMiddleware } from "./exchange-blocklist";+import { providerScrapeController } from "../controllers/v2/scrape-alexandria";import express, { Request, Response } from "express";import { Agent, fetch } from "undici";import { config } from "../config";@@ -134,10 +135,25 @@ exchangeRouter.get(wrap(exchangeProxy(DISCOVER_TIMEOUT_MS)),);+// Provider agreements the web app offers for acceptance; a catalogue read, never an acceptance.+exchangeRouter.get(+ "/provider-terms{/*path}",+ authMiddleware(RateLimiterMode.Labs),+ wrap(exchangeProxy(DISCOVER_TIMEOUT_MS)),+);++exchangeRouter.post(+ "/provider-terms/events",+ authMiddleware(RateLimiterMode.Labs),+ wrap(exchangeProxy(DISCOVER_TIMEOUT_MS)),+);+exchangeRouter.post("/retrieve",authMiddleware(RateLimiterMode.Labs),- wrap(exchangeProxy(RETRIEVE_TIMEOUT_MS)),+ wrap((req, res) =>+ providerScrapeController(req as RequestWithAuth<any, any, any>, res, true),+ ),);exchangeRouter.get(diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex a6d34ce59..89a2c64f9 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -31,6 +31,7 @@ import {import { getTeamBalance } from "../services/autumn/usage";import { getThirdPartyDataTermsRequiredResponse } from "../lib/exchange";import { getExchangeAccessForRequestBody } from "../lib/exchange-request";+import { isToolsOnlySearch } from "../search/alexandria";import { getScrapeZDR } from "../lib/zdr-helpers";import { isAgentInteropSecretValid } from "../lib/agent-interop";@@ -114,6 +115,20 @@ export function checkCreditsMiddleware(// If verified, fall through to normal credit check (key is now on real account)}+ // Tool discovery is free; provider execution reserves its own credits.+ const sources = (req.body as any)?.sources;+ const categories = (req.body as any)?.categories;+ const toolsOnly =+ req.path === "/search" && isToolsOnlySearch(sources, categories);+ if (+ (req.path === "/scrape" &&+ (req.body as any)?.alexandria !== undefined) ||+ toolsOnly+ ) {+ req.account = { remainingCredits: Infinity };+ return next();+ }+if (!minimum && req.body) {minimum = Number((req.body as any)?.limit ?? (req.body as any)?.urls?.length ?? 1,diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.tsnew file mode 100644index 000000000..21f0eff0b--- /dev/null+++ b/apps/api/src/search/alexandria.ts@@ -0,0 +1,282 @@+import { z } from "zod";+import type { Logger } from "winston";+import { exchangeRequest } from "../services/alexandria/client";+import {+ toolSchema,+ type DiscoveredTool,+} from "../services/alexandria/contracts";++export const isAlexandriaSource = (source: { type: string }) =>+ source.type === "alexandria";+type ToolDiscovery = {+ items: DiscoveredTool[];+ warning?: string;+};++type SourceLike = string | { type: string };+type CategoryLike = string | { type: string };++const typeOf = (value: SourceLike | CategoryLike) =>+ typeof value === "string" ? value : value?.type;++export function isToolsOnlySearch(+ sources: unknown,+ categories: unknown,+): boolean {+ if (!Array.isArray(sources) || sources.length === 0) return false;+ if (!sources.every(source => typeOf(source) === "alexandria")) return false;+ if (+ Array.isArray(categories) &&+ categories.some(category => typeOf(category) === "developer")+ )+ return false;+ return true;+}++export async function discoverTools(+ input: {+ teamId: string;+ query?: string;+ urls?: string[];+ limit: number;+ timeoutMs: number;+ },+ logger: Logger,+): Promise<ToolDiscovery> {+ const deadline = Date.now() + Math.min(10000, input.timeoutMs);+ const items = new Map<string, DiscoveredTool>();+ const selected = { semantic: new Set<string>(), domain: new Set<string>() };+ let failed = false;+ const remaining = () => {+ if (Date.now() >= deadline) throw new Error("Tool discovery timed out");+ return deadline - Date.now();+ };+ const lookup = async (options: Record<string, unknown>) => {+ const result = await exchangeRequest({+ teamId: input.teamId,+ path: "/v1/retrieve",+ timeoutMs: remaining(),+ maximumCredits: 0,+ body: {+ provider: "firecrawl",+ capability: "find-tools",+ options: {+ ...options,+ level: "tools",+ expand: ["options", "response", "examples"],+ limit: Math.min(input.limit, 24),+ },+ },+ });+ if (result.status !== 200)+ throw new Error("Tool contract lookup unavailable");+ return z+ .object({+ success: z.literal(true),+ creditsCost: z.literal(0),+ data: z.object({ items: z.array(toolSchema) }),+ })+ .parse(result.body).data.items;+ };+ const merge = (+ tool: z.infer<typeof toolSchema>,+ source: "semantic" | "domain",+ urls: string[] = [],+ ) => {+ const id = `${tool.provider}/${tool.capability}`;+ const previous = items.get(id);+ if (+ !selected[source].has(id) &&+ selected[source].size >= Math.min(input.limit, 24)+ )+ return;+ selected[source].add(id);+ if (previous) {+ items.set(id, {+ ...previous,+ matchedBy: [...new Set([...(previous.matchedBy ?? []), source])],+ matchedUrls: [...new Set([...(previous.matchedUrls ?? []), ...urls])],+ });+ } else {+ items.set(id, {+ ...tool,+ id,+ matchedBy: [source],+ matchedUrls: [...new Set(urls)],+ });+ }+ };+ if (input.query) {+ try {+ const result = await exchangeRequest({+ teamId: input.teamId,+ path: `/v1/discover?q=${encodeURIComponent(input.query)}&limit=${Math.min(input.limit, 24)}`,+ timeoutMs: remaining(),+ });+ if (result.status !== 200)+ throw new Error("Semantic discovery unavailable");+ const hits = z+ .object({+ capabilities: z.array(+ z.object({+ provider: z.string(),+ address: z.string(),+ cohorts: z.array(z.string()).default([]),+ }),+ ),+ })+ .parse(result.body)+ .capabilities.slice(0, Math.min(input.limit, 24));+ // Four concurrent contract lookups, preserving semantic rank.+ for (let i = 0; i < hits.length; i += 4) {+ const contracts = await Promise.all(+ hits.slice(i, i + 4).map(async hit => {+ try {+ if (!hit.cohorts.length)+ return (+ await lookup({+ providers: [hit.provider],+ capabilities: [hit.address],+ })+ )[0];+ const parts = [+ hit.cohorts[0],+ hit.provider,+ ...hit.address.split("/"),+ ];+ if (+ parts.some(part => !/^[a-zA-Z0-9_-][a-zA-Z0-9._-]*$/.test(part))+ )+ throw new Error("Invalid tool identifier");+ const contract = await exchangeRequest({+ teamId: input.teamId,+ path: `/v1/discover/${parts.map(encodeURIComponent).join("/")}`,+ timeoutMs: remaining(),+ });+ if (contract.status !== 200)+ throw new Error("Tool contract unavailable");+ const body = z+ .object({+ label: z.string(),+ whenToUse: z.string(),+ returns: z.unknown(),+ })+ .passthrough()+ .parse(contract.body);+ const tool = toolSchema.parse({+ ...body,+ name: body.label,+ description: body.whenToUse,+ response: body.returns,+ });+ if (+ tool.provider !== hit.provider ||+ tool.capability !== hit.address+ )+ throw new Error("Tool identity mismatch");+ return tool;+ } catch {+ failed = true;+ return undefined;+ }+ }),+ );+ for (const tool of contracts) if (tool) merge(tool, "semantic");+ }+ } catch (error) {+ failed = true;+ logger.warn("Semantic tool discovery unavailable", { error });+ }+ }+ const urls = [...new Set(input.urls ?? [])].filter(value => {+ const url = URL.parse(value);+ return (+ url &&+ ["http:", "https:"].includes(url.protocol) &&+ !url.username &&+ !url.password &&+ value.length <= 8192+ );+ });+ if (urls.length) {+ try {+ // Resolve domains once; Exchange owns matching and provider visibility.+ for (let i = 0; i < urls.length; i += 100) {+ const result = await exchangeRequest({+ teamId: input.teamId,+ path: "/v1/skills/resolve",+ body: { urls: urls.slice(i, i + 100) },+ timeoutMs: remaining(),+ });+ if (result.status !== 200)+ throw new Error("Domain discovery unavailable");+ const matches = z+ .object({+ skills: z.array(+ z.object({+ id: z.string(),+ matchedDomains: z.array(z.string()),+ domainCapabilities: z+ .record(z.string(), z.array(z.string()))+ .optional(),+ }),+ ),+ })+ .parse(result.body).skills;+ const hostOf = (value: string) =>+ new URL(value).hostname.toLowerCase().replace(/\.$/, "");+ for (const match of matches) {+ const matchedUrls = urls.slice(i, i + 100).filter(value => {+ const host = hostOf(value);+ return (+ match.matchedDomains.includes(host) ||+ match.matchedDomains.includes(host.replace(/^www\./, ""))+ );+ });+ const capabilitiesFor = (url: string) => {+ const host = hostOf(url);+ return [+ ...new Set([+ ...(match.domainCapabilities?.[url] ?? []),+ ...(match.domainCapabilities?.[host] ?? []),+ ...(match.domainCapabilities?.[host.replace(/^www\./, "")] ??+ []),+ ]),+ ];+ };+ const capabilities = [+ ...new Set(matchedUrls.flatMap(capabilitiesFor)),+ ];+ if (+ !matchedUrls.length ||+ (match.domainCapabilities && !capabilities.length)+ )+ continue;+ for (const tool of await lookup({+ providers: [match.id],+ ...(capabilities.length ? { capabilities } : {}),+ }))+ if (tool.provider === match.id)+ merge(+ tool,+ "domain",+ match.domainCapabilities+ ? matchedUrls.filter(url =>+ capabilitiesFor(url).includes(tool.capability),+ )+ : matchedUrls,+ );+ }+ }+ } catch (error) {+ failed = true;+ logger.warn("Domain tool discovery unavailable", { error });+ }+ }+ return {+ items: [...items.values()],+ ...(failed+ ? { warning: "Some tool discovery results are unavailable." }+ : {}),+ };+}diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsindex 92b97ca93..bc30d9af6 100644--- a/apps/api/src/search/execute.test.ts+++ b/apps/api/src/search/execute.test.ts@@ -2,6 +2,13 @@ const mocks = vi.hoisted(() => ({search: vi.fn(),searchDeveloperCategory: vi.fn(),checkUrlsAgainstThreatPolicy: vi.fn(),+ discoverTools: vi.fn(),+}));++vi.mock("./alexandria", () => ({+ discoverTools: mocks.discoverTools,+ isAlexandriaSource: (source: { type: string }) =>+ source.type === "alexandria",}));vi.mock("./v2", () => ({ search: mocks.search }));@@ -78,6 +85,35 @@ function options(categories: Array<{ type: string }>) {beforeEach(() => {vi.clearAllMocks();mocks.searchDeveloperCategory.mockResolvedValue([developerResult]);+ mocks.discoverTools.mockResolvedValue({+ status: "available",+ items: [{ id: "fred/series/observations" }],+ });+});++it("tool-only search never runs or bills SERP", async () => {+ const result = await executeSearch(+ { ...options([]), sources: [{ type: "alexandria" }] },+ { ...context, flags: { exchangeRetrieve: true } },+ logger,+ );+ expect(mocks.search).not.toHaveBeenCalled();+ expect(result.searchCredits).toBe(0);+ expect(result.response.tools).toEqual([{ id: "fred/series/observations" }]);+});++it("mixed search bills only normal web results", async () => {+ mocks.search.mockResolvedValue({ web: [developerResult] });+ const result = await executeSearch(+ { ...options([]), sources: [{ type: "web" }, { type: "alexandria" }] },+ { ...context, flags: { exchangeRetrieve: true } },+ logger,+ );+ expect(mocks.search).toHaveBeenCalledWith(+ expect.objectContaining({ type: ["web"] }),+ );+ expect(result.searchCredits).toBe(2);+ expect(result.response.tools).toHaveLength(1);});describe("executeSearch developer category", () => {diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 7ddde049a..86089f5e3 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -1,6 +1,7 @@import type { Logger } from "winston";+import { discoverTools, isAlexandriaSource } from "./alexandria";import { search } from "./v2";-import { SearchV2Response } from "../lib/entities";+import { SearchV2Response, SearchResultType } from "../lib/entities";import {buildSearchQuery,getCategoryFromUrl,@@ -43,6 +44,7 @@ interface SearchOptions {enterprise?: ("default" | "anon" | "zdr")[];scrapeOptions?: ScrapeOptions;highlights?: boolean;+ domainTools?: boolean;timeout: number;}@@ -66,6 +68,7 @@ interface SearchContext {}interface SearchExecuteResult {+ toolsWarning?: string;response: SearchV2Response;totalResultsCount: number;developerResultsCount: number;@@ -106,7 +109,14 @@ export async function executeSearch(logger.info("Searching for results");- const searchTypes = [...new Set(sources.map((s: any) => s.type))];+ const wantsTools = sources.some(isAlexandriaSource);+ const searchTypes = [+ ...new Set(+ sources+ .filter(source => !isAlexandriaSource(source))+ .map(source => source.type as SearchResultType),+ ),+ ];const { query: searchQuery, categoryMap } = buildSearchQuery(query,categories,@@ -124,25 +134,26 @@ export async function executeSearch(): null;- const searchResponse = hasOnlyDeveloperCategory(categories)- ? ({} as SearchV2Response)- : ((await search({- query: searchQuery,- logger,- requestId: context.requestId,- advanced: false,- num_results: num_results_buffer,- tbs: options.tbs,- filter: options.filter,- lang: options.lang,- country: options.country,- location: options.location,- safe: options.safe,- type: searchTypes,- enterprise: options.enterprise,- includeDomains: options.includeDomains,- excludeDomains: options.excludeDomains,- })) as SearchV2Response);+ const searchResponse =+ (wantsTools && !searchTypes.length) || hasOnlyDeveloperCategory(categories)+ ? ({} as SearchV2Response)+ : ((await search({+ query: searchQuery,+ logger,+ requestId: context.requestId,+ advanced: false,+ num_results: num_results_buffer,+ tbs: options.tbs,+ filter: options.filter,+ lang: options.lang,+ country: options.country,+ location: options.location,+ safe: options.safe,+ type: searchTypes,+ enterprise: options.enterprise,+ includeDomains: options.includeDomains,+ excludeDomains: options.excludeDomains,+ })) as SearchV2Response);let developerResults = developerResultsPromise? await developerResultsPromise: [];@@ -230,6 +241,34 @@ export async function executeSearch(const developerResultsCount = developerResults.length;totalResultsCount += developerResultsCount;+ let toolsWarning: string | undefined;++ if (+ flags?.exchangeRetrieve &&+ !zeroDataRetention &&+ !options.enterprise?.some(mode => mode === "zdr" || mode === "anon") &&+ (wantsTools || options.domainTools)+ ) {+ const discovery = await discoverTools(+ {+ teamId,+ limit,+ query: wantsTools ? query : undefined,+ urls:+ options.domainTools === false+ ? []+ : [+ ...(searchResponse.web ?? []),+ ...(searchResponse.news ?? []),+ ...developerResults,+ ].flatMap(item => (item.url ? [item.url] : [])),+ timeoutMs: options.timeout,+ },+ logger,+ );+ searchResponse.tools = discovery.items;+ toolsWarning = discovery.warning;+ }const isZDR = options.enterprise?.includes("zdr");const creditsPerTenResults = isZDR ? 10 : 2;@@ -378,6 +417,7 @@ export async function executeSearch(return {response: searchResponse,+ toolsWarning,totalResultsCount,developerResultsCount,searchCredits,diff --git a/apps/api/src/services/alexandria/README.md b/apps/api/src/services/alexandria/README.mdnew file mode 100644index 000000000..37d2a64f2--- /dev/null+++ b/apps/api/src/services/alexandria/README.md@@ -0,0 +1,81 @@+# Alexandria Search and Scrape++Search discovers tools; Scrape executes them. Both require the+`exchangeRetrieve` flag. Ordinary Search and Scrape keep their paths and billing.++```json+{+ "query": "economic indicators",+ "sources": ["web", "alexandria"],+ "domainTools": true+}+```++Search returns contracts in `data.tools`, up to `limit` per discovery source+(semantic and domain). Discovery is free and never counts toward+`creditsUsed`; web results are billed as before. `domainTools: false` disables+URL matching only.++```json+{+ "alexandria": {+ "provider": "fred",+ "capability": "series/observations",+ "options": { "series_id": "GDP" }+ }+}+```++An ordinary URL scrape accepts `domainTools: true` (default off) and adds+`data.tools` matched to the page's domain in the same shape as Search. It+needs Exchange access and no zero data retention, refused with the same 403+Search returns; discovery itself is free and never fails the scrape.++Scrape with `alexandria` accepts one call or up to ten and returns+`data.alexandria` with `data.creditsCost`. `/exchange/retrieve` shares the path; its single-call+shape relays a provider error with the Exchange's status and `code`.++## Billing++Inline in the request (`retrieve.ts`): authorize through the Exchange+(`/v1/provider-terms/requirements`, which refuses unknown providers) against+the organization's `organizationDataSourceAccess` flags, quote+(`/v1/retrieve/quote`), reserve with Autumn (`lockCredits`, lock+`alexandria_<chargeId>`), execute `/v1/retrieve` with the budget and deadline+headers, settle the receipt (`finalizeCreditsLock`), enqueue the ledger write+on the billing queue with job id `alexandria-bill-<chargeId>`, and report to+`/v1/usage-events/billing`. Paid requests fail closed when authorization,+quote, reservation, or `USE_DB_AUTHENTICATION` is unavailable. No worker,+queue, or migration is needed.++## Idempotency++The charge id is `sha256(teamId, x-request-id)`; send `x-request-id` on every+paid request or a retry is a new charge. One Redis record per charge id lives+seven days:++- completed: replays the response and its `scrape_id`, no second execution+- same id, different payload: 409 `duplicate_request`+- still running: 409 `request_in_flight`+- refused before execution (authorization, quote, hold, Exchange 4xx or+ `deadline_exceeded`): hold released, record dropped, same id may retry. A+ release that does not land is logged; that hold expires on its own shortly+ after the request deadline, so a retry may briefly hold credits twice+- uncertain (Exchange 5xx, timeout, malformed or over-budget receipt, crash+ after the record was written): 503 `request_unresolved`, record kept for+ manual reconciliation, hold expires on its own; the Exchange has no+ request-id idempotency, so this is never re-executed automatically++An unsettled confirm returns the answer, writes no ledger row, and sends no+Exchange confirmation; the run stays pending on the Exchange for+reconciliation. A ledger commit or enqueue that fails is refunded at Autumn on+the direct route; on the firebill route the durable charge stands pending+reconciliation. Provider tools do not support forced zero data retention;+those teams are refused before any record is written.++## Runtime++API, Redis, Exchange (`FIRE_EXCHANGE_URL`; a non-blank+`EXCHANGE_INTERNAL_SECRET` enables the usage report) and Autumn for paid+execution. Paid settlement is covered by mocked tests only and has not run+against an Autumn sandbox.diff --git a/apps/api/src/services/alexandria/access.test.ts b/apps/api/src/services/alexandria/access.test.tsnew file mode 100644index 000000000..1b8e2271b--- /dev/null+++ b/apps/api/src/services/alexandria/access.test.ts@@ -0,0 +1,59 @@+const mocks = vi.hoisted(() => ({ request: vi.fn() }));+vi.mock("../../config", () => ({+ config: { USE_DB_AUTHENTICATION: true, FIRECRAWL_DASHBOARD_URL: "https://d" },+}));+vi.mock("./client", () => ({ exchangeRequest: mocks.request }));+import { authorizeProviders } from "./access";++const calls = [+ { provider: "fred", capability: "series/observations", options: {} },+];+const requirement = (required: boolean) => ({+ status: 200,+ body: {+ providers: [+ {+ provider: "fred",+ required,+ terms: { key: "fred", version: "2026-01" },+ },+ ],+ },+});++beforeEach(() => vi.clearAllMocks());++it("refuses a provider the Exchange does not know before any quote or execution", async () => {+ mocks.request.mockResolvedValue({ status: 404, body: { code: "not_found" } });+ const denied = await authorizeProviders("team", calls, {});+ expect(denied?.status).toBe(404);+ expect(denied?.body).toEqual(+ expect.objectContaining({ code: "unknown_provider" }),+ );+});++it("requires the organization to hold the current agreement when terms are required", async () => {+ mocks.request.mockResolvedValue(requirement(true));+ const stale = {+ organizationDataSourceAccess: {+ fred: { status: "enabled", termsKey: "fred", termsVersion: "2025-01" },+ },+ };+ expect((await authorizeProviders("team", calls, stale))?.status).toBe(403);+ const current = {+ organizationDataSourceAccess: {+ fred: { status: "enabled", termsKey: "fred", termsVersion: "2026-01" },+ },+ };+ expect(await authorizeProviders("team", calls, current)).toBeUndefined();+});++it("refuses a disabled provider and fails closed when agreements are unavailable", async () => {+ mocks.request.mockResolvedValue(requirement(false));+ const disabled = {+ organizationDataSourceAccess: { fred: { status: "disabled" } },+ };+ expect((await authorizeProviders("team", calls, disabled))?.status).toBe(403);+ mocks.request.mockResolvedValue({ status: 503, body: "down" });+ expect((await authorizeProviders("team", calls, {}))?.status).toBe(503);+});diff --git a/apps/api/src/services/alexandria/access.ts b/apps/api/src/services/alexandria/access.tsnew file mode 100644index 000000000..4b20200fa--- /dev/null+++ b/apps/api/src/services/alexandria/access.ts@@ -0,0 +1,74 @@+import { z } from "zod";+import { config } from "../../config";+import type { TeamFlags } from "../../controllers/v2/types";+import { getThirdPartyDataTermsRequiredResponse } from "../../lib/exchange";+import { exchangeRequest } from "./client";+import { refusal, type ExchangeResponse, type ProviderCall } from "./contracts";++const requirementsSchema = z.object({+ providers: z.array(+ z.object({+ provider: z.string(),+ required: z.boolean(),+ terms: z+ .object({ key: z.string(), version: z.string() })+ .passthrough()+ .nullable(),+ }),+ ),+});++export async function authorizeProviders(+ teamId: string,+ calls: ProviderCall[],+ flags: TeamFlags | null | undefined,+): Promise<ExchangeResponse | undefined> {+ const providers = [...new Set(calls.map(call => call.provider))];+ const response = await exchangeRequest({+ teamId,+ path: "/v1/provider-terms/requirements",+ body: { providers },+ timeoutMs: 10000,+ }).catch(() => undefined);+ if (response?.status === 404)+ return refusal(404, "Unknown provider. No provider was executed.", {+ code: "unknown_provider",+ });+ const parsed =+ response?.status === 200+ ? requirementsSchema.safeParse(response.body)+ : undefined;+ const answered = new Set(+ parsed?.success ? parsed.data.providers.map(item => item.provider) : [],+ );+ if (+ !parsed?.success ||+ answered.size !== providers.length ||+ providers.some(provider => !answered.has(provider))+ )+ return refusal(+ 503,+ "Provider agreements are unavailable. No provider was executed.",+ );+ if (config.USE_DB_AUTHENTICATION !== true) return undefined;++ for (const item of parsed.data.providers) {+ const access = flags?.organizationDataSourceAccess?.[item.provider];+ if (access && access.status !== "enabled")+ return refusal(+ 403,+ `Access to ${item.provider} is disabled for this organization.`,+ );+ if (+ item.required &&+ item.terms &&+ (access?.termsKey !== item.terms.key ||+ access?.termsVersion !== item.terms.version)+ )+ return {+ status: 403,+ body: getThirdPartyDataTermsRequiredResponse(item.terms),+ };+ }+ return undefined;+}diff --git a/apps/api/src/services/alexandria/client.ts b/apps/api/src/services/alexandria/client.tsnew file mode 100644index 000000000..b4e0beca0--- /dev/null+++ b/apps/api/src/services/alexandria/client.ts@@ -0,0 +1,60 @@+import { Agent, fetch } from "undici";+import { config } from "../../config";+import type { ExchangeResponse } from "./contracts";++const dispatcher = new Agent({+ connectTimeout: 10000,+ headersTimeout: 50000,+ bodyTimeout: 50000,+});++export async function exchangeRequest(input: {+ teamId: string;+ path: string;+ body?: unknown;+ timeoutMs: number;+ requestId?: string;+ maximumCredits?: number;+}): Promise<ExchangeResponse> {+ if (!config.FIRE_EXCHANGE_URL) throw new Error("Exchange is not configured");+ const base = config.FIRE_EXCHANGE_URL.replace(/\/+$/, "");+ const response = await fetch(base + input.path, {+ method: input.body === undefined ? "GET" : "POST",+ redirect: "manual",+ dispatcher,+ signal: AbortSignal.timeout(Math.max(1, input.timeoutMs)),+ headers: {+ "content-type": "application/json",+ "x-exchange-team-id": input.teamId,+ "x-exchange-extended-catalog-access": "true",+ ...(input.requestId ? { "x-request-id": input.requestId } : {}),+ ...(input.maximumCredits === undefined+ ? {}+ : {+ "x-exchange-max-credits": String(input.maximumCredits),+ // Leave the Exchange a margin to answer before our own timeout.+ "x-exchange-deadline": String(+ Date.now() ++ input.timeoutMs -+ Math.min(2000, input.timeoutMs / 10),+ ),+ }),+ },+ body: input.body === undefined ? undefined : JSON.stringify(input.body),+ });+ // Bound retained provider responses, including bodies without Content-Length.+ const chunks: Uint8Array[] = [];+ let size = 0;+ for await (const chunk of response.body ?? []) {+ size += chunk.length;+ if (size > 5 * 1024 * 1024)+ throw new Error("Exchange response is too large");+ chunks.push(chunk);+ }+ const text = Buffer.concat(chunks).toString();+ let body: unknown = text;+ try {+ body = JSON.parse(text);+ } catch {}+ return { status: response.status, body };+}diff --git a/apps/api/src/services/alexandria/contracts.ts b/apps/api/src/services/alexandria/contracts.tsnew file mode 100644index 000000000..33757a5ee--- /dev/null+++ b/apps/api/src/services/alexandria/contracts.ts@@ -0,0 +1,77 @@+import { z } from "zod";++export const callSchema = z.strictObject({+ provider: z.string().min(1).max(200),+ capability: z.string().min(1).max(200),+ options: z.record(z.string(), z.unknown()).default({}),+});+export const callsSchema = z.array(callSchema).min(1).max(10);+export type ProviderCall = z.infer<typeof callSchema>;++const credits = z.number().int().nonnegative().safe();+const identity = { provider: z.string(), capability: z.string() };+const resultSchema = z.union([+ z+ .object({+ ...identity,+ creditsCost: credits,+ data: z.unknown().refine(value => value !== undefined),+ error: z.never().optional(),+ })+ .passthrough(),+ z+ .object({+ provider: z.string().optional(),+ capability: z.string().optional(),+ creditsCost: z.literal(0).optional(),+ error: z+ .object({+ code: z.string(),+ message: z.string(),+ status: z.number().int().min(100).max(999).optional(),+ })+ .passthrough(),+ })+ .passthrough(),+]);+export const answerSchema = z.object({+ success: z.literal(true),+ creditsCost: credits,+ results: z.array(resultSchema).min(1).max(10),+});++export const toolSchema = z+ .object({+ provider: z.string(),+ capability: z.string(),+ name: z.string(),+ description: z.string(),+ creditsCost: credits,+ perRecord: z.boolean(),+ options: z.array(+ z.object({ name: z.string(), type: z.string() }).passthrough(),+ ),+ response: z+ .object({+ about: z.string(),+ key: z.string(),+ fields: z.array(z.unknown()),+ })+ .passthrough(),+ })+ .passthrough();+export type DiscoveredTool = z.infer<typeof toolSchema> & {+ id: string;+ matchedBy: ("semantic" | "domain")[];+ matchedUrls: string[];+};++export type ExchangeResponse = { status: number; body: unknown };+export const refusal = (+ status: number,+ error: string,+ extra: Record<string, unknown> = {},+): ExchangeResponse => ({+ status,+ body: { success: false, error, ...extra },+});diff --git a/apps/api/src/services/alexandria/retrieve.test.ts b/apps/api/src/services/alexandria/retrieve.test.tsnew file mode 100644index 000000000..d9fd7c4d5--- /dev/null+++ b/apps/api/src/services/alexandria/retrieve.test.ts@@ -0,0 +1,265 @@+const mocks = vi.hoisted(() => {+ const store = new Map<string, string>();+ return {+ store,+ config: { USE_DB_AUTHENTICATION: true, FIRE_EXCHANGE_URL: "https://x" },+ redis: {+ set: vi.fn(async (key: string, value: string, ...args: unknown[]) => {+ if (args.includes("NX") && store.has(key)) return null;+ store.set(key, value);+ return "OK";+ }),+ get: vi.fn(async (key: string) => store.get(key) ?? null),+ del: vi.fn(async (key: string) => (store.delete(key) ? 1 : 0)),+ },+ request: vi.fn(),+ authorize: vi.fn(),+ lock: vi.fn(),+ finalize: vi.fn(),+ refund: vi.fn(),+ billAdd: vi.fn(),+ report: vi.fn(),+ };+});+vi.mock("../../config", () => ({ config: mocks.config }));+vi.mock("../rate-limiter", () => ({ redisRateLimitClient: mocks.redis }));+vi.mock("./client", () => ({ exchangeRequest: mocks.request }));+vi.mock("./access", () => ({ authorizeProviders: mocks.authorize }));+vi.mock("../queue-service", () => ({+ getBillingQueue: () => ({ add: mocks.billAdd }),+}));+vi.mock("../../lib/exchange", () => ({+ reportExchangeUsageBilling: mocks.report,+}));+vi.mock("../autumn/autumn.service", () => ({+ autumnService: {+ lockCredits: mocks.lock,+ finalizeCreditsLock: mocks.finalize,+ refundCredits: mocks.refund,+ isRoutedThroughFirebill: async () => false,+ },+ featureIdForBillingEndpoint: () => "credits",+}));+import { retrieveProviders } from "./retrieve";++const call = {+ provider: "fred",+ capability: "series/observations",+ options: { series_id: "GDP" },+};+const answer = {+ success: true,+ creditsCost: 3,+ results: [{ ...call, creditsCost: 3, data: {} }],+};+const run = (overrides: Record<string, unknown> = {}) =>+ retrieveProviders({+ teamId: "team",+ apiKeyId: 12,+ flags: {},+ calls: [call],+ requestId: "request-1",+ scrapeId: "scrape-1",+ timeoutMs: 50000,+ ...overrides,+ });+const executions = () =>+ mocks.request.mock.calls.filter(([arg]) => arg.path === "/v1/retrieve");+const exchangeAnswers = (+ body: unknown,+ status = 200,+ quote: unknown = { status: 200, body: { maximumCredits: 5 } },+) =>+ mocks.request.mockImplementation(async arg =>+ arg.path.endsWith("/quote") ? quote : { status, body },+ );++beforeEach(() => {+ vi.clearAllMocks();+ mocks.store.clear();+ mocks.config.USE_DB_AUTHENTICATION = true;+ mocks.authorize.mockResolvedValue(undefined);+ mocks.lock.mockResolvedValue({ status: "locked", lockId: "held" });+ mocks.finalize.mockResolvedValue(true);+ mocks.refund.mockResolvedValue(undefined);+ mocks.billAdd.mockResolvedValue({});+ mocks.report.mockResolvedValue(true);+ exchangeAnswers(answer);+});++it("quotes, reserves, executes within budget, settles actual usage, records once, and replays", async () => {+ expect(await run()).toEqual({+ status: 200,+ body: answer,+ executed: true,+ scrapeId: "scrape-1",+ });+ expect(mocks.lock).toHaveBeenCalledWith(+ expect.objectContaining({ value: 5, featureId: "credits" }),+ );+ expect(executions()[0][0]).toEqual(+ expect.objectContaining({+ maximumCredits: 5,+ requestId: expect.any(String),+ }),+ );+ expect(mocks.finalize).toHaveBeenCalledWith(+ expect.objectContaining({+ lockId: "held",+ action: "confirm",+ overrideValue: 3,+ heldValue: 5,+ }),+ );+ expect(mocks.billAdd).toHaveBeenCalledWith(+ "bill_team",+ expect.objectContaining({ credits: 3, autumnTrackInRequest: true }),+ expect.objectContaining({+ jobId: expect.stringMatching(/^alexandria-bill-/),+ }),+ );+ expect(mocks.report).toHaveBeenCalledWith(+ expect.objectContaining({ status: "confirmed" }),+ );++ // The replay keeps the original scrape id and executes nothing.+ expect(await run({ scrapeId: "scrape-2" })).toEqual({+ status: 200,+ body: answer,+ executed: false,+ scrapeId: "scrape-1",+ });+ expect(executions()).toHaveLength(1);+ expect(mocks.finalize).toHaveBeenCalledTimes(1);+ expect(mocks.billAdd).toHaveBeenCalledTimes(1);+});++it("refuses a different payload under the same x-request-id", async () => {+ await run();+ const other = await run({+ calls: [{ ...call, options: { series_id: "CPI" } }],+ });+ expect(other.status).toBe(409);+ expect(other.body).toEqual(+ expect.objectContaining({ code: "duplicate_request" }),+ );+ expect(executions()).toHaveLength(1);+});++it.each([+ [+ "authorization refusal",+ () => mocks.authorize.mockResolvedValueOnce({ status: 403, body: {} }),+ 403,+ ],+ [+ "quote outage",+ () => exchangeAnswers(answer, 200, { status: 503, body: "down" }),+ 503,+ ],+ [+ "malformed quote",+ () => exchangeAnswers(answer, 200, { status: 200, body: { nope: 1 } }),+ 502,+ ],+ [+ "billing not configured",+ () => {+ mocks.config.USE_DB_AUTHENTICATION = false;+ },+ 503,+ ],+ [+ "denied hold",+ () => mocks.lock.mockResolvedValueOnce({ status: "denied" }),+ 402,+ ],+ [+ "skipped hold",+ () => mocks.lock.mockResolvedValueOnce({ status: "skipped" }),+ 503,+ ],+])(+ "fails closed on %s and lets the same id retry",+ async (_, arrange, expected) => {+ arrange();+ const refused = await run();+ expect(refused.status).toBe(expected);+ expect(refused.executed).toBe(false);+ expect(executions()).toHaveLength(0);+ expect(mocks.finalize).not.toHaveBeenCalled();+ expect(mocks.store.size).toBe(0);+ mocks.config.USE_DB_AUTHENTICATION = true;+ exchangeAnswers(answer);+ expect((await run()).status).toBe(200);+ expect(executions()).toHaveLength(1);+ },+);++it("holds an ambiguous execution for reconciliation without settling or re-executing", async () => {+ exchangeAnswers("gateway error", 502);+ const first = await run();+ expect(first.status).toBe(503);+ expect(first.body).toEqual(+ expect.objectContaining({ code: "request_unresolved" }),+ );+ expect(mocks.finalize).not.toHaveBeenCalled();++ exchangeAnswers(answer);+ expect((await run()).body).toEqual(+ expect.objectContaining({ code: "request_unresolved" }),+ );+ expect(executions()).toHaveLength(1);+ expect(mocks.billAdd).not.toHaveBeenCalled();+});++it("releases the hold on a definitive refusal and relays it without caching", async () => {+ exchangeAnswers(+ { code: "credit_budget_exceeded", error: "Over budget." },+ 402,+ );+ expect(await run()).toEqual({+ status: 402,+ body: {+ success: false,+ code: "credit_budget_exceeded",+ error: "Over budget.",+ },+ executed: true,+ scrapeId: "scrape-1",+ });+ expect(mocks.finalize).toHaveBeenCalledWith(+ expect.objectContaining({ lockId: "held", action: "release" }),+ );+ expect(mocks.store.size).toBe(0);+});++it("does not settle a receipt over budget", async () => {+ exchangeAnswers({+ ...answer,+ creditsCost: 6,+ results: [{ ...answer.results[0], creditsCost: 6 }],+ });+ expect((await run()).body).toEqual(+ expect.objectContaining({ code: "request_unresolved" }),+ );+ expect(mocks.finalize).not.toHaveBeenCalled();+ expect(mocks.billAdd).not.toHaveBeenCalled();+});++it("returns the answer but records nothing when the settle does not land", async () => {+ mocks.finalize.mockResolvedValueOnce(false);+ expect((await run()).status).toBe(200);+ expect(mocks.billAdd).not.toHaveBeenCalled();+ expect(mocks.report).not.toHaveBeenCalled();+});++it("refunds the direct-Autumn charge and leaves the Exchange usage pending when the ledger enqueue fails", async () => {+ mocks.billAdd.mockRejectedValue(new Error("queue down"));+ expect((await run()).status).toBe(200);+ expect(mocks.billAdd).toHaveBeenCalledTimes(3);+ expect(mocks.refund).toHaveBeenCalledWith(+ expect.objectContaining({ teamId: "team", value: 3 }),+ );+ expect(mocks.report).not.toHaveBeenCalled();+});diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsnew file mode 100644index 000000000..2c8ca81c9--- /dev/null+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -0,0 +1,410 @@+import { createHash } from "node:crypto";+import { z } from "zod";+import { config } from "../../config";+import type { TeamFlags } from "../../controllers/v2/types";+import { reportExchangeUsageBilling } from "../../lib/exchange";+import { logger } from "../../lib/logger";+import {+ autumnService,+ featureIdForBillingEndpoint,+} from "../autumn/autumn.service";+import { getBillingQueue } from "../queue-service";+import { redisRateLimitClient } from "../rate-limiter";+import { authorizeProviders } from "./access";+import { exchangeRequest } from "./client";+import {+ answerSchema,+ refusal,+ type ExchangeResponse,+ type ProviderCall,+} from "./contracts";++export const REQUEST_ID_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/;+const RETENTION_SECONDS = 7 * 86400;+const GRACE_MS = 30_000;+const HOLD_GRACE_MS = 15 * 60_000;++// One record per charge id (team + x-request-id), written before the provider+// call so a crash is found as unresolved instead of re-executed, and rewritten+// with the response so replays never execute twice.+type Retrieval = {+ fingerprint: string;+ phase: "executing" | "done";+ deadline: number;+ scrapeId: string;+ lockId?: string;+ maximumCredits?: number;+ response?: ExchangeResponse;+ failure?: string;+};++type ProviderRetrieval = ExchangeResponse & {+ executed: boolean;+ scrapeId: string;+};++const canonical = (value: unknown): unknown =>+ Array.isArray(value)+ ? value.map(canonical)+ : value && typeof value === "object"+ ? Object.fromEntries(+ Object.entries(value)+ .sort(([a], [b]) => a.localeCompare(b))+ .map(([key, item]) => [key, canonical(item)]),+ )+ : value;+const hash = (value: unknown) =>+ createHash("sha256")+ .update(JSON.stringify(canonical(value)))+ .digest("hex");++const unresolved = (chargeId: string) =>+ refusal(+ 503,+ "The provider request did not complete and its outcome is uncertain. Keep this x-request-id for reconciliation; do not create a new request.",+ { code: "request_unresolved", chargeId },+ );+const relay = (response: ExchangeResponse): ExchangeResponse => {+ const body = (response.body ?? {}) as Record<string, unknown>;+ return refusal(+ response.status,+ typeof body.error === "string"+ ? body.error+ : "The provider request was refused.",+ typeof body.code === "string" ? { code: body.code } : {},+ );+};++export async function retrieveProviders(input: {+ teamId: string;+ apiKeyId: number | null;+ flags: TeamFlags | null | undefined;+ calls: ProviderCall[];+ requestId: string;+ scrapeId: string;+ timeoutMs: number;+ bypassBilling?: boolean;+}): Promise<ProviderRetrieval> {+ const notExecuted = (response: ExchangeResponse): ProviderRetrieval => ({+ ...response,+ executed: false,+ scrapeId: input.scrapeId,+ });+ if (!REQUEST_ID_PATTERN.test(input.requestId))+ return notExecuted(+ refusal(+ 400,+ "Invalid x-request-id; use 1-128 letters, digits, dots, underscores, colons or hyphens.",+ ),+ );+ if (Buffer.byteLength(JSON.stringify(input.calls)) > 256 * 1024)+ return notExecuted(refusal(400, "Provider options exceed 256 KB."));++ const billable = !input.bypassBilling;+ const id = hash([input.teamId, input.requestId]);+ const key = `alexandria:retrieve:${id}`;+ const fingerprint = hash([input.calls, billable]);+ const record: Retrieval = {+ fingerprint,+ phase: "executing",+ deadline: Date.now() + input.timeoutMs,+ scrapeId: input.scrapeId,+ };+ const write = (next: Retrieval) =>+ redisRateLimitClient.set(+ key,+ JSON.stringify(next),+ "EX",+ RETENTION_SECONDS,+ );++ const claimed = await redisRateLimitClient.set(+ key,+ JSON.stringify(record),+ "EX",+ RETENTION_SECONDS,+ "NX",+ );+ if (claimed !== "OK") {+ const raw = await redisRateLimitClient.get(key);+ const existing: Retrieval | null = raw ? JSON.parse(raw) : null;+ if (existing?.phase === "done" && existing.fingerprint === fingerprint)+ return {+ ...existing.response!,+ executed: false,+ scrapeId: existing.scrapeId ?? input.scrapeId,+ };+ if (existing && existing.fingerprint !== fingerprint)+ return notExecuted(+ refusal(+ 409,+ "This x-request-id already belongs to a different provider request.",+ { code: "duplicate_request", chargeId: id },+ ),+ );+ if (+ existing &&+ !existing.failure &&+ Date.now() <= existing.deadline + GRACE_MS+ )+ return notExecuted(+ refusal(+ 409,+ "This provider request is still running. Retry with the same x-request-id once it has finished.",+ { code: "request_in_flight", chargeId: id },+ ),+ );+ return notExecuted(unresolved(id));+ }++ const featureId = featureIdForBillingEndpoint("scrape");+ const properties = {+ source: "alexandria",+ endpoint: "scrape",+ chargeId: id,+ apiKeyId: input.apiKeyId,+ };+ const remaining = () => Math.max(1, record.deadline - Date.now());+ const refuse = async (response: ExchangeResponse, executed = false) => {+ await redisRateLimitClient.del(key);+ return { ...response, executed, scrapeId: input.scrapeId };+ };++ let lockId: string | undefined;+ let maximumCredits: number;+ let refundable = false;+ try {+ const denied = await authorizeProviders(+ input.teamId,+ input.calls,+ input.flags,+ );+ if (denied) return refuse(denied);++ const quote = await exchangeRequest({+ teamId: input.teamId,+ path: "/v1/retrieve/quote",+ body: { requests: input.calls },+ timeoutMs: Math.min(10000, remaining()),+ }).catch(error => {+ logger.warn("Provider quote unavailable", { error, chargeId: id });+ return undefined;+ });+ if (!quote || quote.status >= 500 || quote.status < 200)+ return refuse(+ refusal(+ 503,+ "Provider quote is unavailable. No provider was executed.",+ ),+ );+ if (quote.status !== 200) return refuse(relay(quote));+ const parsedQuote = z+ .object({ maximumCredits: z.number().int().nonnegative().safe() })+ .safeParse(quote.body);+ if (!parsedQuote.success)+ return refuse(+ refusal(502, "Provider quote was malformed. No provider was executed."),+ );+ maximumCredits = parsedQuote.data.maximumCredits;++ if (billable && maximumCredits > 0) {+ if (!config.USE_DB_AUTHENTICATION)+ return refuse(+ refusal(+ 503,+ "Paid provider billing is not configured. No provider was executed.",+ ),+ );+ refundable = !(await autumnService.isRoutedThroughFirebill(input.teamId));+ const hold = await autumnService.lockCredits({+ teamId: input.teamId,+ value: maximumCredits,+ lockId: `alexandria_${id}`,+ expiresAt: record.deadline + HOLD_GRACE_MS,+ featureId,+ properties,+ });+ if (hold.status === "denied")+ return refuse(+ hold.reason === "gate_unavailable"+ ? refusal(+ 503,+ "Credit reservation is unavailable. No provider was executed.",+ { code: "billing_unavailable" },+ )+ : refusal(+ 402,+ `This request needs up to ${maximumCredits} credits reserved. No provider was executed.`,+ { code: "insufficient_credits" },+ ),+ );+ if (hold.status === "skipped")+ return refuse(+ refusal(+ 503,+ "Credit reservation is unavailable. No provider was executed.",+ ),+ );+ lockId = hold.lockId;+ }+ } catch (error) {+ await redisRateLimitClient.del(key).catch(() => {});+ throw error;+ }+ record.lockId = lockId;+ record.maximumCredits = maximumCredits;+ const fail = async (reason: string): Promise<ProviderRetrieval> => {+ logger.error(+ "Provider request outcome is uncertain; needs reconciliation",+ {+ chargeId: id,+ teamId: input.teamId,+ lockId,+ maximumCredits,+ reason,+ },+ );+ await write({ ...record, failure: reason }).catch(() => {});+ return { ...unresolved(id), executed: true, scrapeId: input.scrapeId };+ };+ const finalize = async (credits: number) => {+ if (!lockId) return true;+ const settled = await autumnService.finalizeCreditsLock({+ teamId: input.teamId,+ lockId,+ action: credits > 0 ? "confirm" : "release",+ ...(credits > 0 ? { overrideValue: credits } : {}),+ heldValue: maximumCredits,+ featureId,+ properties,+ });+ if (!settled)+ logger.error("Provider credits were not settled; hold expires unbilled", {+ chargeId: id,+ teamId: input.teamId,+ lockId,+ credits,+ });+ return settled;+ };++ try {+ await write(record);+ const response = await exchangeRequest({+ teamId: input.teamId,+ path: "/v1/retrieve",+ body: { requests: input.calls },+ timeoutMs: remaining(),+ requestId: id,+ maximumCredits,+ }).catch(error => {+ throw new Error(`Exchange did not answer: ${error?.message ?? error}`);+ });++ const body = (response.body ?? {}) as Record<string, unknown>;+ if (+ (response.status >= 400 && response.status < 500) ||+ (response.status === 504 && body.code === "deadline_exceeded")+ ) {+ await finalize(0);+ return refuse(relay(response), true);+ }+ if (response.status < 200 || response.status >= 300)+ return fail(`Exchange answered ${response.status}`);++ const parsed = answerSchema.safeParse(response.body);+ if (!parsed.success) return fail("Exchange answer was malformed");+ const answer = parsed.data;+ const receiptMatches =+ answer.results.length === input.calls.length &&+ answer.creditsCost <= maximumCredits &&+ answer.results.reduce((sum, item) => sum + (item.creditsCost ?? 0), 0) ===+ answer.creditsCost &&+ answer.results.every(+ (item, i) =>+ (item.provider ?? input.calls[i].provider) ===+ input.calls[i].provider &&+ (item.capability ?? input.calls[i].capability) ===+ input.calls[i].capability,+ );+ if (!receiptMatches)+ return fail("Exchange receipt did not match the request");++ const credits = answer.creditsCost;+ const settled = await finalize(credits);+ const recorded =+ settled && billable && credits > 0+ ? await recordLedgerUsage(+ id,+ input.teamId,+ input.apiKeyId,+ credits,+ refundable,+ )+ : settled;+ if (!billable || recorded)+ void reportExchangeUsageBilling({+ requestId: id,+ status: billable ? "confirmed" : "void",+ billingReference: `alexandria:${id}`,+ });++ const done: ExchangeResponse = { status: 200, body: answer };+ await write({ ...record, phase: "done", response: done });+ return { ...done, executed: true, scrapeId: input.scrapeId };+ } catch (error) {+ return fail(error instanceof Error ? error.message : String(error));+ }+}++async function recordLedgerUsage(+ id: string,+ teamId: string,+ apiKeyId: number | null,+ credits: number,+ refundable: boolean,+): Promise<boolean> {+ for (let attempt = 1; ; attempt++) {+ try {+ await getBillingQueue().add(+ "bill_team",+ {+ team_id: teamId,+ credits,+ billing: { endpoint: "scrape", chargeId: id },+ is_extract: false,+ timestamp: new Date().toISOString(),+ originating_job_id: id,+ api_key_id: apiKeyId,+ autumnTrackInRequest: refundable,+ },+ { jobId: `alexandria-bill-${id}`, priority: 10 },+ );+ return true;+ } catch (error) {+ if (attempt < 3) {+ await new Promise(resolve => setTimeout(resolve, 250 * attempt));+ continue;+ }+ logger.error(+ refundable+ ? "Provider usage could not be queued for the ledger; refunding the Autumn charge and leaving the Exchange usage pending for reconciliation"+ : "Provider usage could not be queued for the ledger; the firebill charge stands and the Exchange usage stays pending for reconciliation",+ { chargeId: id, teamId, credits, error },+ );+ if (refundable)+ await autumnService.refundCredits({+ teamId,+ value: credits,+ featureId: featureIdForBillingEndpoint("scrape"),+ properties: {+ source: "alexandria",+ endpoint: "scrape",+ chargeId: id,+ },+ idempotencyKey: `fc:refund:scrape:${id}`,+ });+ return false;+ }+ }+}