Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): provider discovery through Search and billed execution through Scrape (#4621)

Developers Digestcc06662bdc1b
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index fb1e655ec..8b130f4ae 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -301,6 +301,7 @@ const configSchema = z.object({
// Exchange (routed data sources service)
FIRE_EXCHANGE_URL: z.url().optional(),
+ EXCHANGE_INTERNAL_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),
// Fire Engine
FIRE_ENGINE_BETA_URL: z.string().optional(),
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.ts
new file mode 100644
index 000000000..54ec54d0f
--- /dev/null
+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts
@@ -0,0 +1,137 @@
+import express from "express";
+import request from "supertest";
+const mocks = vi.hoisted(() => ({ retrieve: vi.fn(), log: vi.fn() }));
+vi.mock("../../services/alexandria/retrieve", () => ({
+ REQUEST_ID_PATTERN: /^[A-Za-z0-9._:-]{1,128}$/,
+ retrieveProviders: mocks.retrieve,
+}));
+vi.mock("../../services/logging/log_job", () => ({ logRequest: mocks.log }));
+vi.mock("../../lib/key-restriction", () => ({
+ checkKeyFormatRestriction: async () => ({ allowed: true }),
+}));
+vi.mock("../../lib/agent-interop", () => ({
+ isAgentInteropSecretValid: (value: string) => value === "test-secret",
+}));
+vi.mock("../../config", () => ({
+ config: {
+ FIRE_EXCHANGE_URL: "https://x",
+ AGENT_INTEROP_SECRET: "test-secret",
+ },
+}));
+import { providerScrapeController } from "./scrape-alexandria";
+
+const call = {
+ provider: "fred",
+ capability: "categories/category",
+ options: {},
+};
+const app = express();
+app.use(express.json());
+app.use((req, _res, next) => {
+ Object.assign(req, {
+ auth: { team_id: "team" },
+ acuc: { api_key_id: 12, flags: { exchangeRetrieve: true } },
+ });
+ next();
+});
+app.post("/v2/scrape", (req, res) => providerScrapeController(req as any, res));
+app.post("/exchange/retrieve", (req, res) =>
+ providerScrapeController(req as any, res, true),
+);
+const result = (results: unknown[], executed = true) => ({
+ status: 200,
+ body: { success: true, creditsCost: 0, results },
+ executed,
+ scrapeId: "scrape-1",
+});
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.log.mockResolvedValue(undefined);
+ mocks.retrieve.mockResolvedValue(
+ result([{ ...call, creditsCost: 0, data: {} }]),
+ );
+});
+
+it("returns the Scrape contract, shares identity with the legacy route, and logs once per execution", async () => {
+ const response = await request(app)
+ .post("/v2/scrape")
+ .set("x-request-id", "same-request")
+ .send({ alexandria: call });
+ expect(response.status).toBe(200);
+ expect(response.body).toEqual({
+ success: true,
+ scrape_id: "scrape-1",
+ data: { alexandria: [expect.any(Object)], creditsCost: 0 },
+ });
+ expect(response.headers["x-request-id"]).toBe("same-request");
+ expect(mocks.retrieve).toHaveBeenCalledWith(
+ expect.objectContaining({
+ calls: [call],
+ requestId: "same-request",
+ apiKeyId: 12,
+ }),
+ );
+
+ mocks.retrieve.mockResolvedValue(
+ result([{ ...call, creditsCost: 0, data: {} }], false),
+ );
+ await request(app)
+ .post("/exchange/retrieve")
+ .set("x-request-id", "same-request")
+ .send(call);
+ const [first, second] = mocks.retrieve.mock.calls.map(([arg]) => arg);
+ expect({ ...first, scrapeId: undefined }).toEqual({
+ ...second,
+ scrapeId: undefined,
+ });
+ expect(mocks.log).toHaveBeenCalledTimes(1);
+});
+
+it("relays a failed single legacy call as an error, not a success", async () => {
+ mocks.retrieve.mockResolvedValue(
+ result([
+ {
+ ...call,
+ creditsCost: 0,
+ error: { code: "credential_missing", message: "No key.", status: 503 },
+ },
+ ]),
+ );
+ const response = await request(app).post("/exchange/retrieve").send(call);
+ expect(response.status).toBe(503);
+ expect(response.body).toEqual({
+ success: false,
+ code: "credential_missing",
+ error: "No key.",
+ });
+});
+
+it("only lets trusted agent interop bypass billing, and prefers its request id", async () => {
+ const untrusted = await request(app)
+ .post("/v2/scrape")
+ .send({
+ alexandria: call,
+ __agentInterop: { auth: "wrong", requestId: "a", shouldBill: false },
+ });
+ expect(untrusted.status).toBe(403);
+ expect(mocks.retrieve).not.toHaveBeenCalled();
+
+ const trusted = await request(app)
+ .post("/v2/scrape")
+ .set("x-request-id", "hop-id")
+ .send({
+ alexandria: call,
+ __agentInterop: {
+ auth: "test-secret",
+ requestId: "agent-id",
+ shouldBill: false,
+ boostConcurrency: true,
+ },
+ });
+ expect(trusted.status).toBe(200);
+ expect(mocks.retrieve).toHaveBeenCalledWith(
+ expect.objectContaining({ requestId: "agent-id", bypassBilling: true }),
+ );
+ expect(mocks.log).not.toHaveBeenCalled();
+});
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.ts
new file mode 100644
index 000000000..51f4e63b3
--- /dev/null
+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts
@@ -0,0 +1,174 @@
+import { z } from "zod";
+import type { Response } from "express";
+import { randomUUID } from "node:crypto";
+import { config } from "../../config";
+import { logger } from "../../lib/logger";
+import { isAgentInteropSecretValid } from "../../lib/agent-interop";
+import { externalRequestId } from "../../lib/external-request-id";
+import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { checkKeyFormatRestriction } from "../../lib/key-restriction";
+import { logRequest } from "../../services/logging/log_job";
+import {
+ callsSchema,
+ callSchema,
+ answerSchema,
+} from "../../services/alexandria/contracts";
+import {
+ REQUEST_ID_PATTERN,
+ retrieveProviders,
+} from "../../services/alexandria/retrieve";
+import type { RequestWithAuth } from "./types";
+
+const providerScrapeSchema = z.strictObject({
+ alexandria: z.preprocess(
+ value => (Array.isArray(value) ? value : [value]),
+ callsSchema,
+ ),
+ timeout: z
+ .number()
+ .int()
+ .positive()
+ .default(50000)
+ .transform(value => Math.min(value, 50000)),
+ origin: z.string().default("api"),
+ integration: z.string().nullable().optional(),
+ __agentInterop: z
+ .strictObject({
+ auth: z.string(),
+ requestId: z.string(),
+ shouldBill: z.boolean(),
+ boostConcurrency: z.boolean().optional(),
+ })
+ .optional(),
+});
+
+export async function providerScrapeController(
+ req: RequestWithAuth<any, any, any>,
+ res: Response,
+ legacy = false,
+) {
+ const legacyBody = legacy
+ ? z
+ .union([callSchema, z.strictObject({ requests: callsSchema })])
+ .parse(req.body)
+ : null;
+ const body = providerScrapeSchema.parse(
+ legacyBody
+ ? {
+ alexandria:
+ "requests" in legacyBody ? legacyBody.requests : [legacyBody],
+ }
+ : req.body,
+ );
+
+ if (body.__agentInterop) {
+ if (!config.AGENT_INTEROP_SECRET)
+ return res
+ .status(403)
+ .json({ success: false, error: "Agent interop is not enabled." });
+ if (!isAgentInteropSecretValid(body.__agentInterop.auth))
+ return res
+ .status(403)
+ .json({ success: false, error: "Invalid agent interop." });
+ }
+ const requestId =
+ body.__agentInterop?.requestId ?? req.get("x-request-id") ?? randomUUID();
+ if (REQUEST_ID_PATTERN.test(requestId))
+ res.setHeader("x-request-id", requestId);
+
+ if (!req.acuc?.flags?.exchangeRetrieve)
+ return res.status(403).json({
+ success: false,
+ error: "This endpoint is not enabled for this team.",
+ });
+ if (getScrapeZDR(req.acuc.flags) === "forced")
+ return res.status(403).json({
+ success: false,
+ error: "Provider tools do not support zero data retention.",
+ });
+ if (
+ (req as any).agentIndexOnly ||
+ ["pending", "blocked"].includes(req.acuc._agentSponsor?.status ?? "")
+ )
+ return res.status(403).json({
+ success: false,
+ code: "AGENT_INDEX_ONLY",
+ error: "Verify this API key before executing provider tools.",
+ });
+ const restriction = await checkKeyFormatRestriction(
+ ["json"],
+ [],
+ req.acuc.api_key_id,
+ req.acuc.flags,
+ );
+ if (!restriction.allowed)
+ return res
+ .status(restriction.status)
+ .json({ success: false, error: restriction.error });
+ if (!config.FIRE_EXCHANGE_URL)
+ return res
+ .status(503)
+ .json({ success: false, error: "This endpoint is not available." });
+
+ let result;
+ try {
+ result = await retrieveProviders({
+ teamId: req.auth.team_id,
+ apiKeyId: req.acuc.api_key_id ?? null,
+ flags: req.acuc.flags,
+ calls: body.alexandria,
+ requestId,
+ scrapeId: randomUUID(),
+ timeoutMs: body.timeout,
+ bypassBilling: body.__agentInterop?.shouldBill === false,
+ });
+ } catch (error) {
+ logger.error("Provider scrape unavailable", {
+ error,
+ teamId: req.auth.team_id,
+ });
+ return res.status(503).json({
+ success: false,
+ error: "Provider request unavailable. Retry with the same x-request-id.",
+ });
+ }
+ if (result.executed && !body.__agentInterop)
+ void logRequest({
+ id: result.scrapeId,
+ kind: "scrape",
+ api_version: "v2",
+ external_request_id: externalRequestId(req),
+ team_id: req.auth.team_id,
+ api_key_id: req.acuc.api_key_id ?? null,
+ origin: body.origin,
+ integration: body.integration ?? null,
+ target_hint: `alexandria:${body.alexandria.map(call => `${call.provider}/${call.capability}`).join(",")}`,
+ zeroDataRetention: false,
+ }).catch(error =>
+ logger.warn("Provider request logging failed", {
+ error,
+ scrapeId: result.scrapeId,
+ }),
+ );
+
+ if (result.status !== 200) return res.status(result.status).json(result.body);
+ const answer = answerSchema.parse(result.body);
+ if (legacy) {
+ if (legacyBody && "requests" in legacyBody) return res.json(answer);
+ const first = answer.results[0];
+ if (first.error) {
+ const status = first.error.status ?? 502;
+ return res.status(status >= 400 && status <= 599 ? status : 502).json({
+ success: false,
+ code: first.error.code,
+ error: first.error.message,
+ });
+ }
+ return res.json({ success: true, ...first });
+ }
+ return res.json({
+ success: true,
+ scrape_id: result.scrapeId,
+ data: { alexandria: answer.results, creditsCost: answer.creditsCost },
+ });
+}
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index da7e24665..74940dd1a 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -1,4 +1,6 @@
import { Response } from "express";
+import { providerScrapeController } from "./scrape-alexandria";
+import { discoverTools } from "../../search/alexandria";
import { config } from "../../config";
import { logger as _logger } from "../../lib/logger";
import {
@@ -50,6 +52,8 @@ export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
res: Response<ScrapeResponse>,
) {
+ if (req.body && "alexandria" in req.body)
+ return providerScrapeController(req, res);
// Resolved before the root span starts so the whole request trace stays
// unrecorded for zero-data-retention requests (see otel-tracer).
const zeroDataRetentionTrace =
@@ -152,6 +156,16 @@ export async function scrapeController(
getScrapeZDR(req.acuc?.flags) === "forced" ||
(req.body.zeroDataRetention ?? false) ||
(req.body.lockdown ?? false);
+ if (
+ req.body.domainTools &&
+ (!req.acuc?.flags?.exchangeRetrieve || zeroDataRetention)
+ )
+ return res.status(403).json({
+ success: false,
+ error: !req.acuc?.flags?.exchangeRetrieve
+ ? "The alexandria source is not enabled for this team."
+ : "Provider discovery requires access and does not support zero data retention.",
+ });
const billing: BillingMetadata = req.body.__agentInterop
? { endpoint: "agent" as const, jobId }
: { endpoint: "scrape" as const, jobId };
@@ -625,6 +639,29 @@ export async function scrapeController(
concurrencyLimited,
concurrencyQueueDurationMs: lockTime || undefined,
});
+ const tools =
+ req.body.domainTools && config.FIRE_EXCHANGE_URL
+ ? await discoverTools(
+ {
+ teamId: req.auth.team_id,
+ urls: [
+ ...new Set(
+ [
+ req.body.url,
+ doc!.metadata?.sourceURL,
+ doc!.metadata?.url,
+ ].filter((u): u is string => typeof u === "string"),
+ ),
+ ],
+ limit: 24,
+ timeoutMs: 10000,
+ },
+ logger,
+ ).catch(error => {
+ logger.warn("Domain tool discovery failed", { error });
+ return undefined;
+ })
+ : undefined;
return res.status(200).json({
success: true,
@@ -637,7 +674,9 @@ export async function scrapeController(
? lockTime || 0
: undefined,
},
+ ...(tools ? { tools: tools.items } : {}),
},
+ ...(tools?.warning ? { warning: tools.warning } : {}),
scrape_id: origin?.includes("website") ? jobId : undefined,
});
},
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 41d92d925..3296b4dcd 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -37,6 +37,7 @@ import {
import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { resolveThreatProtection } from "../../lib/threat-protection/request";
+import { isToolsOnlySearch } from "../../search/alexandria";
import {
actionTypesOf,
checkKeyEndpointRestriction,
@@ -129,6 +130,27 @@ async function searchControllerInner(
typeof req.body?.origin === "string" ? req.body.origin : undefined;
req.body = searchRequestSchema.parse(req.body);
+ const wantsTools = req.body.sources.some(
+ source => source.type === "alexandria",
+ );
+ if (wantsTools && !req.body.query.trim())
+ return res.status(400).json({
+ success: false,
+ error: "A query is required for tool search.",
+ });
+ if (
+ (wantsTools || req.body.domainTools) &&
+ (!req.acuc?.flags?.exchangeRetrieve ||
+ teamForcedKind ||
+ req.body.enterprise?.some(mode => mode === "zdr" || mode === "anon"))
+ )
+ return res.status(403).json({
+ success: false,
+ error: !req.acuc?.flags?.exchangeRetrieve
+ ? "The alexandria source is not enabled for this team."
+ : "Provider discovery requires access and does not support zero data retention.",
+ });
+
const requestedFormats = formatTypesOf(req.body.scrapeOptions?.formats);
const keyRestriction = await checkKeyFormatRestriction(
requestedFormats,
@@ -252,8 +274,9 @@ async function searchControllerInner(
});
}
+ const toolsOnly = isToolsOnlySearch(req.body.sources, req.body.categories);
const projectedKeylessCredits =
- !isSearchPreview && shouldBill
+ !isSearchPreview && shouldBill && !toolsOnly
? projectSearchTotalCredits(
{
limit: req.body.limit,
@@ -295,6 +318,7 @@ async function searchControllerInner(
enterprise: req.body.enterprise,
scrapeOptions: req.body.scrapeOptions,
highlights: req.body.highlights,
+ domainTools: req.body.domainTools,
timeout: req.body.timeout,
},
{
@@ -430,6 +454,7 @@ async function searchControllerInner(
data: result.response,
creditsUsed: result.totalCredits,
id: jobId,
+ ...(result.toolsWarning ? { warning: result.toolsWarning } : {}),
});
} catch (error) {
if (reservedKeylessCredits > 0 && !reconciledKeylessCredits) {
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 9b0dd1024..a1e9d421e 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1116,6 +1116,7 @@ const scrapeRequestSchemaBase = baseScrapeOptions.extend({
origin: z.string().optional().prefault("api"),
integration: integrationSchema.optional().transform(val => val || null),
zeroDataRetention: z.boolean().optional(),
+ domainTools: z.boolean().optional(),
__agentInterop: z
.object({
auth: z.string(),
@@ -1538,7 +1539,9 @@ export type ScrapeResponse =
| {
success: true;
warning?: string;
- data: Document;
+ data: Document & {
+ tools?: import("../../services/alexandria/contracts").DiscoveredTool[];
+ };
scrape_id?: string;
};
@@ -1888,6 +1891,7 @@ type Account = {
};
export type TeamFlags = {
+ exchangeRetrieve?: boolean;
ignoreRobots?: "disabled" | "allowed" | "forced";
customRobotsAgent?: "disabled" | "allowed";
threatProtection?: "disabled" | "allowed" | "forced";
@@ -2323,13 +2327,16 @@ export const searchRequestSchema = z
sources: z
.union([
// Array of strings (simple format)
- z.array(z.enum(["web", "images", "news"])),
+ z.array(z.enum(["web", "images", "news", "alexandria"])),
// Array of objects (advanced format)
z.array(
z.union([
webSearchSourceOptions,
imagesSearchSourceOptions,
newsSearchSourceOptions,
+ z.strictObject({
+ type: z.literal("alexandria"),
+ }),
]),
),
])
@@ -2367,6 +2374,7 @@ export const searchRequestSchema = z
// our index. When omitted, the caller integration and rollout cohort decide
// whether generated highlights are returned or only run in shadow mode.
highlights: z.boolean().optional(),
+ domainTools: z.boolean().optional(),
__searchPreviewToken: z.string().optional(),
threatProtection: threatProtectionOverrideSchema.optional(),
scrapeOptions: baseScrapeOptions
diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts
index 0d5dac8ee..440bf134f 100644
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@ -164,6 +164,7 @@ export interface WebSearchResult {
export type SearchResultType = "web" | "images" | "news";
export interface SearchV2Response {
+ tools?: import("../services/alexandria/contracts").DiscoveredTool[];
web?: WebSearchResult[];
images?: ImageSearchResult[];
news?: NewsSearchResult[];
diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.ts
index 6a4f0a69d..cfdd098bc 100644
--- a/apps/api/src/lib/exchange.test.ts
+++ b/apps/api/src/lib/exchange.test.ts
@@ -292,9 +292,7 @@ describe("Exchange routing", () => {
it("accepts only formats the Exchange can return directly", () => {
expect(isSupportedExchangeFormatRequest(undefined)).toBe(true);
- expect(isSupportedExchangeFormatRequest([{ type: "markdown" }])).toBe(
- true,
- );
+ expect(isSupportedExchangeFormatRequest([{ type: "markdown" }])).toBe(true);
expect(isSupportedExchangeFormatRequest(["json"])).toBe(true);
expect(
isSupportedExchangeFormatRequest([
@@ -305,9 +303,9 @@ describe("Exchange routing", () => {
expect(isSupportedExchangeFormatRequest([{ type: "html" }])).toBe(false);
// deterministicJson extractors run against page HTML, which Exchange
// responses do not carry.
- expect(isSupportedExchangeFormatRequest([{ type: "deterministicJson" }])).toBe(
- false,
- );
+ expect(
+ isSupportedExchangeFormatRequest([{ type: "deterministicJson" }]),
+ ).toBe(false);
expect(isSupportedExchangeFormatRequest([])).toBe(false);
});
@@ -403,15 +401,21 @@ describe("Exchange routing", () => {
terms: ACME_TERMS,
});
- expect(getThirdPartyDataTermsRequiredResponse(ACME_TERMS)).toMatchObject({
+ const response = getThirdPartyDataTermsRequiredResponse(ACME_TERMS);
+ expect(response).toMatchObject({
success: false,
code: "THIRD_PARTY_DATA_TERMS_REQUIRED",
requiresAction: {
type: "accept_terms",
terms: "acme",
version: "2026-01-01",
+ url: expect.stringMatching(/\/app\/alexandria\/acme$/),
},
});
+ // The message carries the provider, version and link, since most clients relay only `error`.
+ expect(response.error).toContain("acme");
+ expect(response.error).toContain("2026-01-01");
+ expect(response.error).toContain(response.requiresAction.url);
});
it("requires current terms when the accepted version is stale", async () => {
@@ -562,7 +566,10 @@ describe("Exchange routing", () => {
"https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing",
expect.objectContaining({
method: "POST",
- body: JSON.stringify({ status: "confirmed", billingReference: "bill-1" }),
+ body: JSON.stringify({
+ status: "confirmed",
+ billingReference: "bill-1",
+ }),
}),
);
diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.ts
index 41518418b..937ddd8ac 100644
--- a/apps/api/src/lib/exchange.ts
+++ b/apps/api/src/lib/exchange.ts
@@ -70,8 +70,6 @@ type ExchangeProvider = {
const SUPPORTED_FORMATS = new Set(["markdown", "json"]);
const EXCHANGE_BETA_FLAG = "professionalProfileCompanyDataBeta";
const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED";
-const THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE =
- "An organization admin must accept this data source's terms before this URL can be processed.";
const EXCHANGE_PROVIDERS_PATH = "/v1/providers";
const EXCHANGE_PROVIDERS_TIMEOUT_MS = 2_000;
@@ -515,20 +513,22 @@ export async function canUseExchangeForRequest(
return (await getExchangeAccessForRequest(input)).allowed;
}
-function getThirdPartyDataTermsSettingsUrl(): string {
- return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/settings?tab=data-sources`;
+// Terms are keyed by provider id, so the key doubles as the provider page to accept them on.
+function getThirdPartyDataTermsUrl(terms: ExchangeTerms): string {
+ return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/alexandria/${encodeURIComponent(terms.key)}`;
}
export function getThirdPartyDataTermsRequiredResponse(terms: ExchangeTerms) {
+ const url = getThirdPartyDataTermsUrl(terms);
return {
success: false as const,
code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE as "THIRD_PARTY_DATA_TERMS_REQUIRED",
- error: THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE,
+ error: `An organization admin must accept the ${terms.key} provider's terms (version ${terms.version}) before this request can run. Accept them at ${url}`,
requiresAction: {
type: "accept_terms",
terms: terms.key,
version: terms.version,
- url: getThirdPartyDataTermsSettingsUrl(),
+ url,
},
};
}
@@ -599,24 +599,72 @@ export async function reportExchangeBilling(input: {
return false;
}
+ return deliverBillingReport({
+ url: `${baseUrl}/v1/access-events/${encodeURIComponent(input.accessEventId)}/billing`,
+ headers: { "Content-Type": "application/json" },
+ body: {
+ status: input.status,
+ ...(input.billingReference === undefined
+ ? {}
+ : { billingReference: input.billingReference }),
+ },
+ retryNotFound: false,
+ context: { accessEventId: input.accessEventId, status: input.status },
+ });
+}
+
+/**
+ * Report the billing outcome of a tool execution's usage rows, keyed by the
+ * `x-request-id` sent with `/v1/retrieve`. Internal-secret route; usage is
+ * recorded asynchronously on the Exchange, so a 404 is retried like a 5xx.
+ */
+export async function reportExchangeUsageBilling(input: {
+ requestId: string;
+ status: "confirmed" | "void";
+ billingReference?: string;
+}): Promise<boolean> {
+ const baseUrl = getExchangeBaseUrl();
+ if (!baseUrl || !config.EXCHANGE_INTERNAL_SECRET) {
+ return false;
+ }
+
+ return deliverBillingReport({
+ url: `${baseUrl}/v1/usage-events/billing`,
+ headers: {
+ "Content-Type": "application/json",
+ "x-exchange-secret": config.EXCHANGE_INTERNAL_SECRET,
+ },
+ body: [
+ {
+ requestId: input.requestId,
+ status: input.status,
+ ...(input.billingReference === undefined
+ ? {}
+ : { billingReference: input.billingReference }),
+ },
+ ],
+ retryNotFound: true,
+ context: { requestId: input.requestId, status: input.status },
+ });
+}
+
+async function deliverBillingReport(input: {
+ url: string;
+ headers: Record<string, string>;
+ body: unknown;
+ retryNotFound: boolean;
+ context: Record<string, unknown>;
+}): Promise<boolean> {
for (let attempt = 1; attempt <= EXCHANGE_BILLING_ATTEMPTS; attempt++) {
let retryAfterMs: number | undefined;
try {
- const response = await fetch(
- `${baseUrl}/v1/access-events/${encodeURIComponent(input.accessEventId)}/billing`,
- {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({
- status: input.status,
- ...(input.billingReference === undefined
- ? {}
- : { billingReference: input.billingReference }),
- }),
- signal: AbortSignal.timeout(EXCHANGE_BILLING_TIMEOUT_MS),
- },
- );
+ const response = await fetch(input.url, {
+ method: "POST",
+ headers: input.headers,
+ body: JSON.stringify(input.body),
+ signal: AbortSignal.timeout(EXCHANGE_BILLING_TIMEOUT_MS),
+ });
if (response.ok) {
return true;
@@ -624,11 +672,15 @@ export async function reportExchangeBilling(input: {
// 4xx responses other than 429 are definitive (conflict, unknown
// event) - the Exchange has spoken and a retry cannot change the
- // answer. 429 is transient rate limiting and retries.
- if (response.status < 500 && response.status !== 429) {
+ // answer. 429 is transient rate limiting and retries, as does a 404
+ // where the caller knows the rows are written asynchronously.
+ if (
+ response.status < 500 &&
+ response.status !== 429 &&
+ !(input.retryNotFound && response.status === 404)
+ ) {
rootLogger.warn("Exchange billing report rejected", {
- accessEventId: input.accessEventId,
- status: input.status,
+ ...input.context,
statusCode: response.status,
});
return false;
@@ -639,15 +691,13 @@ export async function reportExchangeBilling(input: {
}
rootLogger.warn("Exchange billing report failed", {
- accessEventId: input.accessEventId,
- status: input.status,
+ ...input.context,
statusCode: response.status,
attempt,
});
} catch (error) {
rootLogger.warn("Exchange billing report errored", {
- accessEventId: input.accessEventId,
- status: input.status,
+ ...input.context,
attempt,
error,
});
diff --git a/apps/api/src/routes/exchange.ts b/apps/api/src/routes/exchange.ts
index 31b31e988..cc5709fed 100644
--- a/apps/api/src/routes/exchange.ts
+++ b/apps/api/src/routes/exchange.ts
@@ -1,4 +1,5 @@
import { bountyBlocklistMiddleware } from "./exchange-blocklist";
+import { providerScrapeController } from "../controllers/v2/scrape-alexandria";
import express, { Request, Response } from "express";
import { Agent, fetch } from "undici";
import { config } from "../config";
@@ -134,10 +135,25 @@ exchangeRouter.get(
wrap(exchangeProxy(DISCOVER_TIMEOUT_MS)),
);
+// Provider agreements the web app offers for acceptance; a catalogue read, never an acceptance.
+exchangeRouter.get(
+ "/provider-terms{/*path}",
+ authMiddleware(RateLimiterMode.Labs),
+ wrap(exchangeProxy(DISCOVER_TIMEOUT_MS)),
+);
+
+exchangeRouter.post(
+ "/provider-terms/events",
+ authMiddleware(RateLimiterMode.Labs),
+ wrap(exchangeProxy(DISCOVER_TIMEOUT_MS)),
+);
+
exchangeRouter.post(
"/retrieve",
authMiddleware(RateLimiterMode.Labs),
- wrap(exchangeProxy(RETRIEVE_TIMEOUT_MS)),
+ wrap((req, res) =>
+ providerScrapeController(req as RequestWithAuth<any, any, any>, res, true),
+ ),
);
exchangeRouter.get(
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index a6d34ce59..89a2c64f9 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -31,6 +31,7 @@ import {
import { getTeamBalance } from "../services/autumn/usage";
import { getThirdPartyDataTermsRequiredResponse } from "../lib/exchange";
import { getExchangeAccessForRequestBody } from "../lib/exchange-request";
+import { isToolsOnlySearch } from "../search/alexandria";
import { getScrapeZDR } from "../lib/zdr-helpers";
import { isAgentInteropSecretValid } from "../lib/agent-interop";
@@ -114,6 +115,20 @@ export function checkCreditsMiddleware(
// If verified, fall through to normal credit check (key is now on real account)
}
+ // Tool discovery is free; provider execution reserves its own credits.
+ const sources = (req.body as any)?.sources;
+ const categories = (req.body as any)?.categories;
+ const toolsOnly =
+ req.path === "/search" && isToolsOnlySearch(sources, categories);
+ if (
+ (req.path === "/scrape" &&
+ (req.body as any)?.alexandria !== undefined) ||
+ toolsOnly
+ ) {
+ req.account = { remainingCredits: Infinity };
+ return next();
+ }
+
if (!minimum && req.body) {
minimum = Number(
(req.body as any)?.limit ?? (req.body as any)?.urls?.length ?? 1,
diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.ts
new file mode 100644
index 000000000..21f0eff0b
--- /dev/null
+++ b/apps/api/src/search/alexandria.ts
@@ -0,0 +1,282 @@
+import { z } from "zod";
+import type { Logger } from "winston";
+import { exchangeRequest } from "../services/alexandria/client";
+import {
+ toolSchema,
+ type DiscoveredTool,
+} from "../services/alexandria/contracts";
+
+export const isAlexandriaSource = (source: { type: string }) =>
+ source.type === "alexandria";
+type ToolDiscovery = {
+ items: DiscoveredTool[];
+ warning?: string;
+};
+
+type SourceLike = string | { type: string };
+type CategoryLike = string | { type: string };
+
+const typeOf = (value: SourceLike | CategoryLike) =>
+ typeof value === "string" ? value : value?.type;
+
+export function isToolsOnlySearch(
+ sources: unknown,
+ categories: unknown,
+): boolean {
+ if (!Array.isArray(sources) || sources.length === 0) return false;
+ if (!sources.every(source => typeOf(source) === "alexandria")) return false;
+ if (
+ Array.isArray(categories) &&
+ categories.some(category => typeOf(category) === "developer")
+ )
+ return false;
+ return true;
+}
+
+export async function discoverTools(
+ input: {
+ teamId: string;
+ query?: string;
+ urls?: string[];
+ limit: number;
+ timeoutMs: number;
+ },
+ logger: Logger,
+): Promise<ToolDiscovery> {
+ const deadline = Date.now() + Math.min(10000, input.timeoutMs);
+ const items = new Map<string, DiscoveredTool>();
+ const selected = { semantic: new Set<string>(), domain: new Set<string>() };
+ let failed = false;
+ const remaining = () => {
+ if (Date.now() >= deadline) throw new Error("Tool discovery timed out");
+ return deadline - Date.now();
+ };
+ const lookup = async (options: Record<string, unknown>) => {
+ const result = await exchangeRequest({
+ teamId: input.teamId,
+ path: "/v1/retrieve",
+ timeoutMs: remaining(),
+ maximumCredits: 0,
+ body: {
+ provider: "firecrawl",
+ capability: "find-tools",
+ options: {
+ ...options,
+ level: "tools",
+ expand: ["options", "response", "examples"],
+ limit: Math.min(input.limit, 24),
+ },
+ },
+ });
+ if (result.status !== 200)
+ throw new Error("Tool contract lookup unavailable");
+ return z
+ .object({
+ success: z.literal(true),
+ creditsCost: z.literal(0),
+ data: z.object({ items: z.array(toolSchema) }),
+ })
+ .parse(result.body).data.items;
+ };
+ const merge = (
+ tool: z.infer<typeof toolSchema>,
+ source: "semantic" | "domain",
+ urls: string[] = [],
+ ) => {
+ const id = `${tool.provider}/${tool.capability}`;
+ const previous = items.get(id);
+ if (
+ !selected[source].has(id) &&
+ selected[source].size >= Math.min(input.limit, 24)
+ )
+ return;
+ selected[source].add(id);
+ if (previous) {
+ items.set(id, {
+ ...previous,
+ matchedBy: [...new Set([...(previous.matchedBy ?? []), source])],
+ matchedUrls: [...new Set([...(previous.matchedUrls ?? []), ...urls])],
+ });
+ } else {
+ items.set(id, {
+ ...tool,
+ id,
+ matchedBy: [source],
+ matchedUrls: [...new Set(urls)],
+ });
+ }
+ };
+ if (input.query) {
+ try {
+ const result = await exchangeRequest({
+ teamId: input.teamId,
+ path: `/v1/discover?q=${encodeURIComponent(input.query)}&limit=${Math.min(input.limit, 24)}`,
+ timeoutMs: remaining(),
+ });
+ if (result.status !== 200)
+ throw new Error("Semantic discovery unavailable");
+ const hits = z
+ .object({
+ capabilities: z.array(
+ z.object({
+ provider: z.string(),
+ address: z.string(),
+ cohorts: z.array(z.string()).default([]),
+ }),
+ ),
+ })
+ .parse(result.body)
+ .capabilities.slice(0, Math.min(input.limit, 24));
+ // Four concurrent contract lookups, preserving semantic rank.
+ for (let i = 0; i < hits.length; i += 4) {
+ const contracts = await Promise.all(
+ hits.slice(i, i + 4).map(async hit => {
+ try {
+ if (!hit.cohorts.length)
+ return (
+ await lookup({
+ providers: [hit.provider],
+ capabilities: [hit.address],
+ })
+ )[0];
+ const parts = [
+ hit.cohorts[0],
+ hit.provider,
+ ...hit.address.split("/"),
+ ];
+ if (
+ parts.some(part => !/^[a-zA-Z0-9_-][a-zA-Z0-9._-]*$/.test(part))
+ )
+ throw new Error("Invalid tool identifier");
+ const contract = await exchangeRequest({
+ teamId: input.teamId,
+ path: `/v1/discover/${parts.map(encodeURIComponent).join("/")}`,
+ timeoutMs: remaining(),
+ });
+ if (contract.status !== 200)
+ throw new Error("Tool contract unavailable");
+ const body = z
+ .object({
+ label: z.string(),
+ whenToUse: z.string(),
+ returns: z.unknown(),
+ })
+ .passthrough()
+ .parse(contract.body);
+ const tool = toolSchema.parse({
+ ...body,
+ name: body.label,
+ description: body.whenToUse,
+ response: body.returns,
+ });
+ if (
+ tool.provider !== hit.provider ||
+ tool.capability !== hit.address
+ )
+ throw new Error("Tool identity mismatch");
+ return tool;
+ } catch {
+ failed = true;
+ return undefined;
+ }
+ }),
+ );
+ for (const tool of contracts) if (tool) merge(tool, "semantic");
+ }
+ } catch (error) {
+ failed = true;
+ logger.warn("Semantic tool discovery unavailable", { error });
+ }
+ }
+ const urls = [...new Set(input.urls ?? [])].filter(value => {
+ const url = URL.parse(value);
+ return (
+ url &&
+ ["http:", "https:"].includes(url.protocol) &&
+ !url.username &&
+ !url.password &&
+ value.length <= 8192
+ );
+ });
+ if (urls.length) {
+ try {
+ // Resolve domains once; Exchange owns matching and provider visibility.
+ for (let i = 0; i < urls.length; i += 100) {
+ const result = await exchangeRequest({
+ teamId: input.teamId,
+ path: "/v1/skills/resolve",
+ body: { urls: urls.slice(i, i + 100) },
+ timeoutMs: remaining(),
+ });
+ if (result.status !== 200)
+ throw new Error("Domain discovery unavailable");
+ const matches = z
+ .object({
+ skills: z.array(
+ z.object({
+ id: z.string(),
+ matchedDomains: z.array(z.string()),
+ domainCapabilities: z
+ .record(z.string(), z.array(z.string()))
+ .optional(),
+ }),
+ ),
+ })
+ .parse(result.body).skills;
+ const hostOf = (value: string) =>
+ new URL(value).hostname.toLowerCase().replace(/\.$/, "");
+ for (const match of matches) {
+ const matchedUrls = urls.slice(i, i + 100).filter(value => {
+ const host = hostOf(value);
+ return (
+ match.matchedDomains.includes(host) ||
+ match.matchedDomains.includes(host.replace(/^www\./, ""))
+ );
+ });
+ const capabilitiesFor = (url: string) => {
+ const host = hostOf(url);
+ return [
+ ...new Set([
+ ...(match.domainCapabilities?.[url] ?? []),
+ ...(match.domainCapabilities?.[host] ?? []),
+ ...(match.domainCapabilities?.[host.replace(/^www\./, "")] ??
+ []),
+ ]),
+ ];
+ };
+ const capabilities = [
+ ...new Set(matchedUrls.flatMap(capabilitiesFor)),
+ ];
+ if (
+ !matchedUrls.length ||
+ (match.domainCapabilities && !capabilities.length)
+ )
+ continue;
+ for (const tool of await lookup({
+ providers: [match.id],
+ ...(capabilities.length ? { capabilities } : {}),
+ }))
+ if (tool.provider === match.id)
+ merge(
+ tool,
+ "domain",
+ match.domainCapabilities
+ ? matchedUrls.filter(url =>
+ capabilitiesFor(url).includes(tool.capability),
+ )
+ : matchedUrls,
+ );
+ }
+ }
+ } catch (error) {
+ failed = true;
+ logger.warn("Domain tool discovery unavailable", { error });
+ }
+ }
+ return {
+ items: [...items.values()],
+ ...(failed
+ ? { warning: "Some tool discovery results are unavailable." }
+ : {}),
+ };
+}
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.ts
index 92b97ca93..bc30d9af6 100644
--- a/apps/api/src/search/execute.test.ts
+++ b/apps/api/src/search/execute.test.ts
@@ -2,6 +2,13 @@ const mocks = vi.hoisted(() => ({
search: vi.fn(),
searchDeveloperCategory: vi.fn(),
checkUrlsAgainstThreatPolicy: vi.fn(),
+ discoverTools: vi.fn(),
+}));
+
+vi.mock("./alexandria", () => ({
+ discoverTools: mocks.discoverTools,
+ isAlexandriaSource: (source: { type: string }) =>
+ source.type === "alexandria",
}));
vi.mock("./v2", () => ({ search: mocks.search }));
@@ -78,6 +85,35 @@ function options(categories: Array<{ type: string }>) {
beforeEach(() => {
vi.clearAllMocks();
mocks.searchDeveloperCategory.mockResolvedValue([developerResult]);
+ mocks.discoverTools.mockResolvedValue({
+ status: "available",
+ items: [{ id: "fred/series/observations" }],
+ });
+});
+
+it("tool-only search never runs or bills SERP", async () => {
+ const result = await executeSearch(
+ { ...options([]), sources: [{ type: "alexandria" }] },
+ { ...context, flags: { exchangeRetrieve: true } },
+ logger,
+ );
+ expect(mocks.search).not.toHaveBeenCalled();
+ expect(result.searchCredits).toBe(0);
+ expect(result.response.tools).toEqual([{ id: "fred/series/observations" }]);
+});
+
+it("mixed search bills only normal web results", async () => {
+ mocks.search.mockResolvedValue({ web: [developerResult] });
+ const result = await executeSearch(
+ { ...options([]), sources: [{ type: "web" }, { type: "alexandria" }] },
+ { ...context, flags: { exchangeRetrieve: true } },
+ logger,
+ );
+ expect(mocks.search).toHaveBeenCalledWith(
+ expect.objectContaining({ type: ["web"] }),
+ );
+ expect(result.searchCredits).toBe(2);
+ expect(result.response.tools).toHaveLength(1);
});
describe("executeSearch developer category", () => {
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 7ddde049a..86089f5e3 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -1,6 +1,7 @@
import type { Logger } from "winston";
+import { discoverTools, isAlexandriaSource } from "./alexandria";
import { search } from "./v2";
-import { SearchV2Response } from "../lib/entities";
+import { SearchV2Response, SearchResultType } from "../lib/entities";
import {
buildSearchQuery,
getCategoryFromUrl,
@@ -43,6 +44,7 @@ interface SearchOptions {
enterprise?: ("default" | "anon" | "zdr")[];
scrapeOptions?: ScrapeOptions;
highlights?: boolean;
+ domainTools?: boolean;
timeout: number;
}
@@ -66,6 +68,7 @@ interface SearchContext {
}
interface SearchExecuteResult {
+ toolsWarning?: string;
response: SearchV2Response;
totalResultsCount: number;
developerResultsCount: number;
@@ -106,7 +109,14 @@ export async function executeSearch(
logger.info("Searching for results");
- const searchTypes = [...new Set(sources.map((s: any) => s.type))];
+ const wantsTools = sources.some(isAlexandriaSource);
+ const searchTypes = [
+ ...new Set(
+ sources
+ .filter(source => !isAlexandriaSource(source))
+ .map(source => source.type as SearchResultType),
+ ),
+ ];
const { query: searchQuery, categoryMap } = buildSearchQuery(
query,
categories,
@@ -124,25 +134,26 @@ export async function executeSearch(
)
: null;
- const searchResponse = hasOnlyDeveloperCategory(categories)
- ? ({} as SearchV2Response)
- : ((await search({
- query: searchQuery,
- logger,
- requestId: context.requestId,
- advanced: false,
- num_results: num_results_buffer,
- tbs: options.tbs,
- filter: options.filter,
- lang: options.lang,
- country: options.country,
- location: options.location,
- safe: options.safe,
- type: searchTypes,
- enterprise: options.enterprise,
- includeDomains: options.includeDomains,
- excludeDomains: options.excludeDomains,
- })) as SearchV2Response);
+ const searchResponse =
+ (wantsTools && !searchTypes.length) || hasOnlyDeveloperCategory(categories)
+ ? ({} as SearchV2Response)
+ : ((await search({
+ query: searchQuery,
+ logger,
+ requestId: context.requestId,
+ advanced: false,
+ num_results: num_results_buffer,
+ tbs: options.tbs,
+ filter: options.filter,
+ lang: options.lang,
+ country: options.country,
+ location: options.location,
+ safe: options.safe,
+ type: searchTypes,
+ enterprise: options.enterprise,
+ includeDomains: options.includeDomains,
+ excludeDomains: options.excludeDomains,
+ })) as SearchV2Response);
let developerResults = developerResultsPromise
? await developerResultsPromise
: [];
@@ -230,6 +241,34 @@ export async function executeSearch(
const developerResultsCount = developerResults.length;
totalResultsCount += developerResultsCount;
+ let toolsWarning: string | undefined;
+
+ if (
+ flags?.exchangeRetrieve &&
+ !zeroDataRetention &&
+ !options.enterprise?.some(mode => mode === "zdr" || mode === "anon") &&
+ (wantsTools || options.domainTools)
+ ) {
+ const discovery = await discoverTools(
+ {
+ teamId,
+ limit,
+ query: wantsTools ? query : undefined,
+ urls:
+ options.domainTools === false
+ ? []
+ : [
+ ...(searchResponse.web ?? []),
+ ...(searchResponse.news ?? []),
+ ...developerResults,
+ ].flatMap(item => (item.url ? [item.url] : [])),
+ timeoutMs: options.timeout,
+ },
+ logger,
+ );
+ searchResponse.tools = discovery.items;
+ toolsWarning = discovery.warning;
+ }
const isZDR = options.enterprise?.includes("zdr");
const creditsPerTenResults = isZDR ? 10 : 2;
@@ -378,6 +417,7 @@ export async function executeSearch(
return {
response: searchResponse,
+ toolsWarning,
totalResultsCount,
developerResultsCount,
searchCredits,
diff --git a/apps/api/src/services/alexandria/README.md b/apps/api/src/services/alexandria/README.md
new file mode 100644
index 000000000..37d2a64f2
--- /dev/null
+++ b/apps/api/src/services/alexandria/README.md
@@ -0,0 +1,81 @@
+# Alexandria Search and Scrape
+
+Search discovers tools; Scrape executes them. Both require the
+`exchangeRetrieve` flag. Ordinary Search and Scrape keep their paths and billing.
+
+```json
+{
+ "query": "economic indicators",
+ "sources": ["web", "alexandria"],
+ "domainTools": true
+}
+```
+
+Search returns contracts in `data.tools`, up to `limit` per discovery source
+(semantic and domain). Discovery is free and never counts toward
+`creditsUsed`; web results are billed as before. `domainTools: false` disables
+URL matching only.
+
+```json
+{
+ "alexandria": {
+ "provider": "fred",
+ "capability": "series/observations",
+ "options": { "series_id": "GDP" }
+ }
+}
+```
+
+An ordinary URL scrape accepts `domainTools: true` (default off) and adds
+`data.tools` matched to the page's domain in the same shape as Search. It
+needs Exchange access and no zero data retention, refused with the same 403
+Search returns; discovery itself is free and never fails the scrape.
+
+Scrape with `alexandria` accepts one call or up to ten and returns
+`data.alexandria` with `data.creditsCost`. `/exchange/retrieve` shares the path; its single-call
+shape relays a provider error with the Exchange's status and `code`.
+
+## Billing
+
+Inline in the request (`retrieve.ts`): authorize through the Exchange
+(`/v1/provider-terms/requirements`, which refuses unknown providers) against
+the organization's `organizationDataSourceAccess` flags, quote
+(`/v1/retrieve/quote`), reserve with Autumn (`lockCredits`, lock
+`alexandria_<chargeId>`), execute `/v1/retrieve` with the budget and deadline
+headers, settle the receipt (`finalizeCreditsLock`), enqueue the ledger write
+on the billing queue with job id `alexandria-bill-<chargeId>`, and report to
+`/v1/usage-events/billing`. Paid requests fail closed when authorization,
+quote, reservation, or `USE_DB_AUTHENTICATION` is unavailable. No worker,
+queue, or migration is needed.
+
+## Idempotency
+
+The charge id is `sha256(teamId, x-request-id)`; send `x-request-id` on every
+paid request or a retry is a new charge. One Redis record per charge id lives
+seven days:
+
+- completed: replays the response and its `scrape_id`, no second execution
+- same id, different payload: 409 `duplicate_request`
+- still running: 409 `request_in_flight`
+- refused before execution (authorization, quote, hold, Exchange 4xx or
+ `deadline_exceeded`): hold released, record dropped, same id may retry. A
+ release that does not land is logged; that hold expires on its own shortly
+ after the request deadline, so a retry may briefly hold credits twice
+- uncertain (Exchange 5xx, timeout, malformed or over-budget receipt, crash
+ after the record was written): 503 `request_unresolved`, record kept for
+ manual reconciliation, hold expires on its own; the Exchange has no
+ request-id idempotency, so this is never re-executed automatically
+
+An unsettled confirm returns the answer, writes no ledger row, and sends no
+Exchange confirmation; the run stays pending on the Exchange for
+reconciliation. A ledger commit or enqueue that fails is refunded at Autumn on
+the direct route; on the firebill route the durable charge stands pending
+reconciliation. Provider tools do not support forced zero data retention;
+those teams are refused before any record is written.
+
+## Runtime
+
+API, Redis, Exchange (`FIRE_EXCHANGE_URL`; a non-blank
+`EXCHANGE_INTERNAL_SECRET` enables the usage report) and Autumn for paid
+execution. Paid settlement is covered by mocked tests only and has not run
+against an Autumn sandbox.
diff --git a/apps/api/src/services/alexandria/access.test.ts b/apps/api/src/services/alexandria/access.test.ts
new file mode 100644
index 000000000..1b8e2271b
--- /dev/null
+++ b/apps/api/src/services/alexandria/access.test.ts
@@ -0,0 +1,59 @@
+const mocks = vi.hoisted(() => ({ request: vi.fn() }));
+vi.mock("../../config", () => ({
+ config: { USE_DB_AUTHENTICATION: true, FIRECRAWL_DASHBOARD_URL: "https://d" },
+}));
+vi.mock("./client", () => ({ exchangeRequest: mocks.request }));
+import { authorizeProviders } from "./access";
+
+const calls = [
+ { provider: "fred", capability: "series/observations", options: {} },
+];
+const requirement = (required: boolean) => ({
+ status: 200,
+ body: {
+ providers: [
+ {
+ provider: "fred",
+ required,
+ terms: { key: "fred", version: "2026-01" },
+ },
+ ],
+ },
+});
+
+beforeEach(() => vi.clearAllMocks());
+
+it("refuses a provider the Exchange does not know before any quote or execution", async () => {
+ mocks.request.mockResolvedValue({ status: 404, body: { code: "not_found" } });
+ const denied = await authorizeProviders("team", calls, {});
+ expect(denied?.status).toBe(404);
+ expect(denied?.body).toEqual(
+ expect.objectContaining({ code: "unknown_provider" }),
+ );
+});
+
+it("requires the organization to hold the current agreement when terms are required", async () => {
+ mocks.request.mockResolvedValue(requirement(true));
+ const stale = {
+ organizationDataSourceAccess: {
+ fred: { status: "enabled", termsKey: "fred", termsVersion: "2025-01" },
+ },
+ };
+ expect((await authorizeProviders("team", calls, stale))?.status).toBe(403);
+ const current = {
+ organizationDataSourceAccess: {
+ fred: { status: "enabled", termsKey: "fred", termsVersion: "2026-01" },
+ },
+ };
+ expect(await authorizeProviders("team", calls, current)).toBeUndefined();
+});
+
+it("refuses a disabled provider and fails closed when agreements are unavailable", async () => {
+ mocks.request.mockResolvedValue(requirement(false));
+ const disabled = {
+ organizationDataSourceAccess: { fred: { status: "disabled" } },
+ };
+ expect((await authorizeProviders("team", calls, disabled))?.status).toBe(403);
+ mocks.request.mockResolvedValue({ status: 503, body: "down" });
+ expect((await authorizeProviders("team", calls, {}))?.status).toBe(503);
+});
diff --git a/apps/api/src/services/alexandria/access.ts b/apps/api/src/services/alexandria/access.ts
new file mode 100644
index 000000000..4b20200fa
--- /dev/null
+++ b/apps/api/src/services/alexandria/access.ts
@@ -0,0 +1,74 @@
+import { z } from "zod";
+import { config } from "../../config";
+import type { TeamFlags } from "../../controllers/v2/types";
+import { getThirdPartyDataTermsRequiredResponse } from "../../lib/exchange";
+import { exchangeRequest } from "./client";
+import { refusal, type ExchangeResponse, type ProviderCall } from "./contracts";
+
+const requirementsSchema = z.object({
+ providers: z.array(
+ z.object({
+ provider: z.string(),
+ required: z.boolean(),
+ terms: z
+ .object({ key: z.string(), version: z.string() })
+ .passthrough()
+ .nullable(),
+ }),
+ ),
+});
+
+export async function authorizeProviders(
+ teamId: string,
+ calls: ProviderCall[],
+ flags: TeamFlags | null | undefined,
+): Promise<ExchangeResponse | undefined> {
+ const providers = [...new Set(calls.map(call => call.provider))];
+ const response = await exchangeRequest({
+ teamId,
+ path: "/v1/provider-terms/requirements",
+ body: { providers },
+ timeoutMs: 10000,
+ }).catch(() => undefined);
+ if (response?.status === 404)
+ return refusal(404, "Unknown provider. No provider was executed.", {
+ code: "unknown_provider",
+ });
+ const parsed =
+ response?.status === 200
+ ? requirementsSchema.safeParse(response.body)
+ : undefined;
+ const answered = new Set(
+ parsed?.success ? parsed.data.providers.map(item => item.provider) : [],
+ );
+ if (
+ !parsed?.success ||
+ answered.size !== providers.length ||
+ providers.some(provider => !answered.has(provider))
+ )
+ return refusal(
+ 503,
+ "Provider agreements are unavailable. No provider was executed.",
+ );
+ if (config.USE_DB_AUTHENTICATION !== true) return undefined;
+
+ for (const item of parsed.data.providers) {
+ const access = flags?.organizationDataSourceAccess?.[item.provider];
+ if (access && access.status !== "enabled")
+ return refusal(
+ 403,
+ `Access to ${item.provider} is disabled for this organization.`,
+ );
+ if (
+ item.required &&
+ item.terms &&
+ (access?.termsKey !== item.terms.key ||
+ access?.termsVersion !== item.terms.version)
+ )
+ return {
+ status: 403,
+ body: getThirdPartyDataTermsRequiredResponse(item.terms),
+ };
+ }
+ return undefined;
+}
diff --git a/apps/api/src/services/alexandria/client.ts b/apps/api/src/services/alexandria/client.ts
new file mode 100644
index 000000000..b4e0beca0
--- /dev/null
+++ b/apps/api/src/services/alexandria/client.ts
@@ -0,0 +1,60 @@
+import { Agent, fetch } from "undici";
+import { config } from "../../config";
+import type { ExchangeResponse } from "./contracts";
+
+const dispatcher = new Agent({
+ connectTimeout: 10000,
+ headersTimeout: 50000,
+ bodyTimeout: 50000,
+});
+
+export async function exchangeRequest(input: {
+ teamId: string;
+ path: string;
+ body?: unknown;
+ timeoutMs: number;
+ requestId?: string;
+ maximumCredits?: number;
+}): Promise<ExchangeResponse> {
+ if (!config.FIRE_EXCHANGE_URL) throw new Error("Exchange is not configured");
+ const base = config.FIRE_EXCHANGE_URL.replace(/\/+$/, "");
+ const response = await fetch(base + input.path, {
+ method: input.body === undefined ? "GET" : "POST",
+ redirect: "manual",
+ dispatcher,
+ signal: AbortSignal.timeout(Math.max(1, input.timeoutMs)),
+ headers: {
+ "content-type": "application/json",
+ "x-exchange-team-id": input.teamId,
+ "x-exchange-extended-catalog-access": "true",
+ ...(input.requestId ? { "x-request-id": input.requestId } : {}),
+ ...(input.maximumCredits === undefined
+ ? {}
+ : {
+ "x-exchange-max-credits": String(input.maximumCredits),
+ // Leave the Exchange a margin to answer before our own timeout.
+ "x-exchange-deadline": String(
+ Date.now() +
+ input.timeoutMs -
+ Math.min(2000, input.timeoutMs / 10),
+ ),
+ }),
+ },
+ body: input.body === undefined ? undefined : JSON.stringify(input.body),
+ });
+ // Bound retained provider responses, including bodies without Content-Length.
+ const chunks: Uint8Array[] = [];
+ let size = 0;
+ for await (const chunk of response.body ?? []) {
+ size += chunk.length;
+ if (size > 5 * 1024 * 1024)
+ throw new Error("Exchange response is too large");
+ chunks.push(chunk);
+ }
+ const text = Buffer.concat(chunks).toString();
+ let body: unknown = text;
+ try {
+ body = JSON.parse(text);
+ } catch {}
+ return { status: response.status, body };
+}
diff --git a/apps/api/src/services/alexandria/contracts.ts b/apps/api/src/services/alexandria/contracts.ts
new file mode 100644
index 000000000..33757a5ee
--- /dev/null
+++ b/apps/api/src/services/alexandria/contracts.ts
@@ -0,0 +1,77 @@
+import { z } from "zod";
+
+export const callSchema = z.strictObject({
+ provider: z.string().min(1).max(200),
+ capability: z.string().min(1).max(200),
+ options: z.record(z.string(), z.unknown()).default({}),
+});
+export const callsSchema = z.array(callSchema).min(1).max(10);
+export type ProviderCall = z.infer<typeof callSchema>;
+
+const credits = z.number().int().nonnegative().safe();
+const identity = { provider: z.string(), capability: z.string() };
+const resultSchema = z.union([
+ z
+ .object({
+ ...identity,
+ creditsCost: credits,
+ data: z.unknown().refine(value => value !== undefined),
+ error: z.never().optional(),
+ })
+ .passthrough(),
+ z
+ .object({
+ provider: z.string().optional(),
+ capability: z.string().optional(),
+ creditsCost: z.literal(0).optional(),
+ error: z
+ .object({
+ code: z.string(),
+ message: z.string(),
+ status: z.number().int().min(100).max(999).optional(),
+ })
+ .passthrough(),
+ })
+ .passthrough(),
+]);
+export const answerSchema = z.object({
+ success: z.literal(true),
+ creditsCost: credits,
+ results: z.array(resultSchema).min(1).max(10),
+});
+
+export const toolSchema = z
+ .object({
+ provider: z.string(),
+ capability: z.string(),
+ name: z.string(),
+ description: z.string(),
+ creditsCost: credits,
+ perRecord: z.boolean(),
+ options: z.array(
+ z.object({ name: z.string(), type: z.string() }).passthrough(),
+ ),
+ response: z
+ .object({
+ about: z.string(),
+ key: z.string(),
+ fields: z.array(z.unknown()),
+ })
+ .passthrough(),
+ })
+ .passthrough();
+export type DiscoveredTool = z.infer<typeof toolSchema> & {
+ id: string;
+ matchedBy: ("semantic" | "domain")[];
+ matchedUrls: string[];
+};
+
+export type ExchangeResponse = { status: number; body: unknown };
+export const refusal = (
+ status: number,
+ error: string,
+ extra: Record<string, unknown> = {},
+): ExchangeResponse => ({
+ status,
+ body: { success: false, error, ...extra },
+});
diff --git a/apps/api/src/services/alexandria/retrieve.test.ts b/apps/api/src/services/alexandria/retrieve.test.ts
new file mode 100644
index 000000000..d9fd7c4d5
--- /dev/null
+++ b/apps/api/src/services/alexandria/retrieve.test.ts
@@ -0,0 +1,265 @@
+const mocks = vi.hoisted(() => {
+ const store = new Map<string, string>();
+ return {
+ store,
+ config: { USE_DB_AUTHENTICATION: true, FIRE_EXCHANGE_URL: "https://x" },
+ redis: {
+ set: vi.fn(async (key: string, value: string, ...args: unknown[]) => {
+ if (args.includes("NX") && store.has(key)) return null;
+ store.set(key, value);
+ return "OK";
+ }),
+ get: vi.fn(async (key: string) => store.get(key) ?? null),
+ del: vi.fn(async (key: string) => (store.delete(key) ? 1 : 0)),
+ },
+ request: vi.fn(),
+ authorize: vi.fn(),
+ lock: vi.fn(),
+ finalize: vi.fn(),
+ refund: vi.fn(),
+ billAdd: vi.fn(),
+ report: vi.fn(),
+ };
+});
+vi.mock("../../config", () => ({ config: mocks.config }));
+vi.mock("../rate-limiter", () => ({ redisRateLimitClient: mocks.redis }));
+vi.mock("./client", () => ({ exchangeRequest: mocks.request }));
+vi.mock("./access", () => ({ authorizeProviders: mocks.authorize }));
+vi.mock("../queue-service", () => ({
+ getBillingQueue: () => ({ add: mocks.billAdd }),
+}));
+vi.mock("../../lib/exchange", () => ({
+ reportExchangeUsageBilling: mocks.report,
+}));
+vi.mock("../autumn/autumn.service", () => ({
+ autumnService: {
+ lockCredits: mocks.lock,
+ finalizeCreditsLock: mocks.finalize,
+ refundCredits: mocks.refund,
+ isRoutedThroughFirebill: async () => false,
+ },
+ featureIdForBillingEndpoint: () => "credits",
+}));
+import { retrieveProviders } from "./retrieve";
+
+const call = {
+ provider: "fred",
+ capability: "series/observations",
+ options: { series_id: "GDP" },
+};
+const answer = {
+ success: true,
+ creditsCost: 3,
+ results: [{ ...call, creditsCost: 3, data: {} }],
+};
+const run = (overrides: Record<string, unknown> = {}) =>
+ retrieveProviders({
+ teamId: "team",
+ apiKeyId: 12,
+ flags: {},
+ calls: [call],
+ requestId: "request-1",
+ scrapeId: "scrape-1",
+ timeoutMs: 50000,
+ ...overrides,
+ });
+const executions = () =>
+ mocks.request.mock.calls.filter(([arg]) => arg.path === "/v1/retrieve");
+const exchangeAnswers = (
+ body: unknown,
+ status = 200,
+ quote: unknown = { status: 200, body: { maximumCredits: 5 } },
+) =>
+ mocks.request.mockImplementation(async arg =>
+ arg.path.endsWith("/quote") ? quote : { status, body },
+ );
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.store.clear();
+ mocks.config.USE_DB_AUTHENTICATION = true;
+ mocks.authorize.mockResolvedValue(undefined);
+ mocks.lock.mockResolvedValue({ status: "locked", lockId: "held" });
+ mocks.finalize.mockResolvedValue(true);
+ mocks.refund.mockResolvedValue(undefined);
+ mocks.billAdd.mockResolvedValue({});
+ mocks.report.mockResolvedValue(true);
+ exchangeAnswers(answer);
+});
+
+it("quotes, reserves, executes within budget, settles actual usage, records once, and replays", async () => {
+ expect(await run()).toEqual({
+ status: 200,
+ body: answer,
+ executed: true,
+ scrapeId: "scrape-1",
+ });
+ expect(mocks.lock).toHaveBeenCalledWith(
+ expect.objectContaining({ value: 5, featureId: "credits" }),
+ );
+ expect(executions()[0][0]).toEqual(
+ expect.objectContaining({
+ maximumCredits: 5,
+ requestId: expect.any(String),
+ }),
+ );
+ expect(mocks.finalize).toHaveBeenCalledWith(
+ expect.objectContaining({
+ lockId: "held",
+ action: "confirm",
+ overrideValue: 3,
+ heldValue: 5,
+ }),
+ );
+ expect(mocks.billAdd).toHaveBeenCalledWith(
+ "bill_team",
+ expect.objectContaining({ credits: 3, autumnTrackInRequest: true }),
+ expect.objectContaining({
+ jobId: expect.stringMatching(/^alexandria-bill-/),
+ }),
+ );
+ expect(mocks.report).toHaveBeenCalledWith(
+ expect.objectContaining({ status: "confirmed" }),
+ );
+
+ // The replay keeps the original scrape id and executes nothing.
+ expect(await run({ scrapeId: "scrape-2" })).toEqual({
+ status: 200,
+ body: answer,
+ executed: false,
+ scrapeId: "scrape-1",
+ });
+ expect(executions()).toHaveLength(1);
+ expect(mocks.finalize).toHaveBeenCalledTimes(1);
+ expect(mocks.billAdd).toHaveBeenCalledTimes(1);
+});
+
+it("refuses a different payload under the same x-request-id", async () => {
+ await run();
+ const other = await run({
+ calls: [{ ...call, options: { series_id: "CPI" } }],
+ });
+ expect(other.status).toBe(409);
+ expect(other.body).toEqual(
+ expect.objectContaining({ code: "duplicate_request" }),
+ );
+ expect(executions()).toHaveLength(1);
+});
+
+it.each([
+ [
+ "authorization refusal",
+ () => mocks.authorize.mockResolvedValueOnce({ status: 403, body: {} }),
+ 403,
+ ],
+ [
+ "quote outage",
+ () => exchangeAnswers(answer, 200, { status: 503, body: "down" }),
+ 503,
+ ],
+ [
+ "malformed quote",
+ () => exchangeAnswers(answer, 200, { status: 200, body: { nope: 1 } }),
+ 502,
+ ],
+ [
+ "billing not configured",
+ () => {
+ mocks.config.USE_DB_AUTHENTICATION = false;
+ },
+ 503,
+ ],
+ [
+ "denied hold",
+ () => mocks.lock.mockResolvedValueOnce({ status: "denied" }),
+ 402,
+ ],
+ [
+ "skipped hold",
+ () => mocks.lock.mockResolvedValueOnce({ status: "skipped" }),
+ 503,
+ ],
+])(
+ "fails closed on %s and lets the same id retry",
+ async (_, arrange, expected) => {
+ arrange();
+ const refused = await run();
+ expect(refused.status).toBe(expected);
+ expect(refused.executed).toBe(false);
+ expect(executions()).toHaveLength(0);
+ expect(mocks.finalize).not.toHaveBeenCalled();
+ expect(mocks.store.size).toBe(0);
+ mocks.config.USE_DB_AUTHENTICATION = true;
+ exchangeAnswers(answer);
+ expect((await run()).status).toBe(200);
+ expect(executions()).toHaveLength(1);
+ },
+);
+
+it("holds an ambiguous execution for reconciliation without settling or re-executing", async () => {
+ exchangeAnswers("gateway error", 502);
+ const first = await run();
+ expect(first.status).toBe(503);
+ expect(first.body).toEqual(
+ expect.objectContaining({ code: "request_unresolved" }),
+ );
+ expect(mocks.finalize).not.toHaveBeenCalled();
+
+ exchangeAnswers(answer);
+ expect((await run()).body).toEqual(
+ expect.objectContaining({ code: "request_unresolved" }),
+ );
+ expect(executions()).toHaveLength(1);
+ expect(mocks.billAdd).not.toHaveBeenCalled();
+});
+
+it("releases the hold on a definitive refusal and relays it without caching", async () => {
+ exchangeAnswers(
+ { code: "credit_budget_exceeded", error: "Over budget." },
+ 402,
+ );
+ expect(await run()).toEqual({
+ status: 402,
+ body: {
+ success: false,
+ code: "credit_budget_exceeded",
+ error: "Over budget.",
+ },
+ executed: true,
+ scrapeId: "scrape-1",
+ });
+ expect(mocks.finalize).toHaveBeenCalledWith(
+ expect.objectContaining({ lockId: "held", action: "release" }),
+ );
+ expect(mocks.store.size).toBe(0);
+});
+
+it("does not settle a receipt over budget", async () => {
+ exchangeAnswers({
+ ...answer,
+ creditsCost: 6,
+ results: [{ ...answer.results[0], creditsCost: 6 }],
+ });
+ expect((await run()).body).toEqual(
+ expect.objectContaining({ code: "request_unresolved" }),
+ );
+ expect(mocks.finalize).not.toHaveBeenCalled();
+ expect(mocks.billAdd).not.toHaveBeenCalled();
+});
+
+it("returns the answer but records nothing when the settle does not land", async () => {
+ mocks.finalize.mockResolvedValueOnce(false);
+ expect((await run()).status).toBe(200);
+ expect(mocks.billAdd).not.toHaveBeenCalled();
+ expect(mocks.report).not.toHaveBeenCalled();
+});
+
+it("refunds the direct-Autumn charge and leaves the Exchange usage pending when the ledger enqueue fails", async () => {
+ mocks.billAdd.mockRejectedValue(new Error("queue down"));
+ expect((await run()).status).toBe(200);
+ expect(mocks.billAdd).toHaveBeenCalledTimes(3);
+ expect(mocks.refund).toHaveBeenCalledWith(
+ expect.objectContaining({ teamId: "team", value: 3 }),
+ );
+ expect(mocks.report).not.toHaveBeenCalled();
+});
diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.ts
new file mode 100644
index 000000000..2c8ca81c9
--- /dev/null
+++ b/apps/api/src/services/alexandria/retrieve.ts
@@ -0,0 +1,410 @@
+import { createHash } from "node:crypto";
+import { z } from "zod";
+import { config } from "../../config";
+import type { TeamFlags } from "../../controllers/v2/types";
+import { reportExchangeUsageBilling } from "../../lib/exchange";
+import { logger } from "../../lib/logger";
+import {
+ autumnService,
+ featureIdForBillingEndpoint,
+} from "../autumn/autumn.service";
+import { getBillingQueue } from "../queue-service";
+import { redisRateLimitClient } from "../rate-limiter";
+import { authorizeProviders } from "./access";
+import { exchangeRequest } from "./client";
+import {
+ answerSchema,
+ refusal,
+ type ExchangeResponse,
+ type ProviderCall,
+} from "./contracts";
+
+export const REQUEST_ID_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/;
+const RETENTION_SECONDS = 7 * 86400;
+const GRACE_MS = 30_000;
+const HOLD_GRACE_MS = 15 * 60_000;
+
+// One record per charge id (team + x-request-id), written before the provider
+// call so a crash is found as unresolved instead of re-executed, and rewritten
+// with the response so replays never execute twice.
+type Retrieval = {
+ fingerprint: string;
+ phase: "executing" | "done";
+ deadline: number;
+ scrapeId: string;
+ lockId?: string;
+ maximumCredits?: number;
+ response?: ExchangeResponse;
+ failure?: string;
+};
+
+type ProviderRetrieval = ExchangeResponse & {
+ executed: boolean;
+ scrapeId: string;
+};
+
+const canonical = (value: unknown): unknown =>
+ Array.isArray(value)
+ ? value.map(canonical)
+ : value && typeof value === "object"
+ ? Object.fromEntries(
+ Object.entries(value)
+ .sort(([a], [b]) => a.localeCompare(b))
+ .map(([key, item]) => [key, canonical(item)]),
+ )
+ : value;
+const hash = (value: unknown) =>
+ createHash("sha256")
+ .update(JSON.stringify(canonical(value)))
+ .digest("hex");
+
+const unresolved = (chargeId: string) =>
+ refusal(
+ 503,
+ "The provider request did not complete and its outcome is uncertain. Keep this x-request-id for reconciliation; do not create a new request.",
+ { code: "request_unresolved", chargeId },
+ );
+const relay = (response: ExchangeResponse): ExchangeResponse => {
+ const body = (response.body ?? {}) as Record<string, unknown>;
+ return refusal(
+ response.status,
+ typeof body.error === "string"
+ ? body.error
+ : "The provider request was refused.",
+ typeof body.code === "string" ? { code: body.code } : {},
+ );
+};
+
+export async function retrieveProviders(input: {
+ teamId: string;
+ apiKeyId: number | null;
+ flags: TeamFlags | null | undefined;
+ calls: ProviderCall[];
+ requestId: string;
+ scrapeId: string;
+ timeoutMs: number;
+ bypassBilling?: boolean;
+}): Promise<ProviderRetrieval> {
+ const notExecuted = (response: ExchangeResponse): ProviderRetrieval => ({
+ ...response,
+ executed: false,
+ scrapeId: input.scrapeId,
+ });
+ if (!REQUEST_ID_PATTERN.test(input.requestId))
+ return notExecuted(
+ refusal(
+ 400,
+ "Invalid x-request-id; use 1-128 letters, digits, dots, underscores, colons or hyphens.",
+ ),
+ );
+ if (Buffer.byteLength(JSON.stringify(input.calls)) > 256 * 1024)
+ return notExecuted(refusal(400, "Provider options exceed 256 KB."));
+
+ const billable = !input.bypassBilling;
+ const id = hash([input.teamId, input.requestId]);
+ const key = `alexandria:retrieve:${id}`;
+ const fingerprint = hash([input.calls, billable]);
+ const record: Retrieval = {
+ fingerprint,
+ phase: "executing",
+ deadline: Date.now() + input.timeoutMs,
+ scrapeId: input.scrapeId,
+ };
+ const write = (next: Retrieval) =>
+ redisRateLimitClient.set(
+ key,
+ JSON.stringify(next),
+ "EX",
+ RETENTION_SECONDS,
+ );
+
+ const claimed = await redisRateLimitClient.set(
+ key,
+ JSON.stringify(record),
+ "EX",
+ RETENTION_SECONDS,
+ "NX",
+ );
+ if (claimed !== "OK") {
+ const raw = await redisRateLimitClient.get(key);
+ const existing: Retrieval | null = raw ? JSON.parse(raw) : null;
+ if (existing?.phase === "done" && existing.fingerprint === fingerprint)
+ return {
+ ...existing.response!,
+ executed: false,
+ scrapeId: existing.scrapeId ?? input.scrapeId,
+ };
+ if (existing && existing.fingerprint !== fingerprint)
+ return notExecuted(
+ refusal(
+ 409,
+ "This x-request-id already belongs to a different provider request.",
+ { code: "duplicate_request", chargeId: id },
+ ),
+ );
+ if (
+ existing &&
+ !existing.failure &&
+ Date.now() <= existing.deadline + GRACE_MS
+ )
+ return notExecuted(
+ refusal(
+ 409,
+ "This provider request is still running. Retry with the same x-request-id once it has finished.",
+ { code: "request_in_flight", chargeId: id },
+ ),
+ );
+ return notExecuted(unresolved(id));
+ }
+
+ const featureId = featureIdForBillingEndpoint("scrape");
+ const properties = {
+ source: "alexandria",
+ endpoint: "scrape",
+ chargeId: id,
+ apiKeyId: input.apiKeyId,
+ };
+ const remaining = () => Math.max(1, record.deadline - Date.now());
+ const refuse = async (response: ExchangeResponse, executed = false) => {
+ await redisRateLimitClient.del(key);
+ return { ...response, executed, scrapeId: input.scrapeId };
+ };
+
+ let lockId: string | undefined;
+ let maximumCredits: number;
+ let refundable = false;
+ try {
+ const denied = await authorizeProviders(
+ input.teamId,
+ input.calls,
+ input.flags,
+ );
+ if (denied) return refuse(denied);
+
+ const quote = await exchangeRequest({
+ teamId: input.teamId,
+ path: "/v1/retrieve/quote",
+ body: { requests: input.calls },
+ timeoutMs: Math.min(10000, remaining()),
+ }).catch(error => {
+ logger.warn("Provider quote unavailable", { error, chargeId: id });
+ return undefined;
+ });
+ if (!quote || quote.status >= 500 || quote.status < 200)
+ return refuse(
+ refusal(
+ 503,
+ "Provider quote is unavailable. No provider was executed.",
+ ),
+ );
+ if (quote.status !== 200) return refuse(relay(quote));
+ const parsedQuote = z
+ .object({ maximumCredits: z.number().int().nonnegative().safe() })
+ .safeParse(quote.body);
+ if (!parsedQuote.success)
+ return refuse(
+ refusal(502, "Provider quote was malformed. No provider was executed."),
+ );
+ maximumCredits = parsedQuote.data.maximumCredits;
+
+ if (billable && maximumCredits > 0) {
+ if (!config.USE_DB_AUTHENTICATION)
+ return refuse(
+ refusal(
+ 503,
+ "Paid provider billing is not configured. No provider was executed.",
+ ),
+ );
+ refundable = !(await autumnService.isRoutedThroughFirebill(input.teamId));
+ const hold = await autumnService.lockCredits({
+ teamId: input.teamId,
+ value: maximumCredits,
+ lockId: `alexandria_${id}`,
+ expiresAt: record.deadline + HOLD_GRACE_MS,
+ featureId,
+ properties,
+ });
+ if (hold.status === "denied")
+ return refuse(
+ hold.reason === "gate_unavailable"
+ ? refusal(
+ 503,
+ "Credit reservation is unavailable. No provider was executed.",
+ { code: "billing_unavailable" },
+ )
+ : refusal(
+ 402,
+ `This request needs up to ${maximumCredits} credits reserved. No provider was executed.`,
+ { code: "insufficient_credits" },
+ ),
+ );
+ if (hold.status === "skipped")
+ return refuse(
+ refusal(
+ 503,
+ "Credit reservation is unavailable. No provider was executed.",
+ ),
+ );
+ lockId = hold.lockId;
+ }
+ } catch (error) {
+ await redisRateLimitClient.del(key).catch(() => {});
+ throw error;
+ }
+ record.lockId = lockId;
+ record.maximumCredits = maximumCredits;
+ const fail = async (reason: string): Promise<ProviderRetrieval> => {
+ logger.error(
+ "Provider request outcome is uncertain; needs reconciliation",
+ {
+ chargeId: id,
+ teamId: input.teamId,
+ lockId,
+ maximumCredits,
+ reason,
+ },
+ );
+ await write({ ...record, failure: reason }).catch(() => {});
+ return { ...unresolved(id), executed: true, scrapeId: input.scrapeId };
+ };
+ const finalize = async (credits: number) => {
+ if (!lockId) return true;
+ const settled = await autumnService.finalizeCreditsLock({
+ teamId: input.teamId,
+ lockId,
+ action: credits > 0 ? "confirm" : "release",
+ ...(credits > 0 ? { overrideValue: credits } : {}),
+ heldValue: maximumCredits,
+ featureId,
+ properties,
+ });
+ if (!settled)
+ logger.error("Provider credits were not settled; hold expires unbilled", {
+ chargeId: id,
+ teamId: input.teamId,
+ lockId,
+ credits,
+ });
+ return settled;
+ };
+
+ try {
+ await write(record);
+ const response = await exchangeRequest({
+ teamId: input.teamId,
+ path: "/v1/retrieve",
+ body: { requests: input.calls },
+ timeoutMs: remaining(),
+ requestId: id,
+ maximumCredits,
+ }).catch(error => {
+ throw new Error(`Exchange did not answer: ${error?.message ?? error}`);
+ });
+
+ const body = (response.body ?? {}) as Record<string, unknown>;
+ if (
+ (response.status >= 400 && response.status < 500) ||
+ (response.status === 504 && body.code === "deadline_exceeded")
+ ) {
+ await finalize(0);
+ return refuse(relay(response), true);
+ }
+ if (response.status < 200 || response.status >= 300)
+ return fail(`Exchange answered ${response.status}`);
+
+ const parsed = answerSchema.safeParse(response.body);
+ if (!parsed.success) return fail("Exchange answer was malformed");
+ const answer = parsed.data;
+ const receiptMatches =
+ answer.results.length === input.calls.length &&
+ answer.creditsCost <= maximumCredits &&
+ answer.results.reduce((sum, item) => sum + (item.creditsCost ?? 0), 0) ===
+ answer.creditsCost &&
+ answer.results.every(
+ (item, i) =>
+ (item.provider ?? input.calls[i].provider) ===
+ input.calls[i].provider &&
+ (item.capability ?? input.calls[i].capability) ===
+ input.calls[i].capability,
+ );
+ if (!receiptMatches)
+ return fail("Exchange receipt did not match the request");
+
+ const credits = answer.creditsCost;
+ const settled = await finalize(credits);
+ const recorded =
+ settled && billable && credits > 0
+ ? await recordLedgerUsage(
+ id,
+ input.teamId,
+ input.apiKeyId,
+ credits,
+ refundable,
+ )
+ : settled;
+ if (!billable || recorded)
+ void reportExchangeUsageBilling({
+ requestId: id,
+ status: billable ? "confirmed" : "void",
+ billingReference: `alexandria:${id}`,
+ });
+
+ const done: ExchangeResponse = { status: 200, body: answer };
+ await write({ ...record, phase: "done", response: done });
+ return { ...done, executed: true, scrapeId: input.scrapeId };
+ } catch (error) {
+ return fail(error instanceof Error ? error.message : String(error));
+ }
+}
+
+async function recordLedgerUsage(
+ id: string,
+ teamId: string,
+ apiKeyId: number | null,
+ credits: number,
+ refundable: boolean,
+): Promise<boolean> {
+ for (let attempt = 1; ; attempt++) {
+ try {
+ await getBillingQueue().add(
+ "bill_team",
+ {
+ team_id: teamId,
+ credits,
+ billing: { endpoint: "scrape", chargeId: id },
+ is_extract: false,
+ timestamp: new Date().toISOString(),
+ originating_job_id: id,
+ api_key_id: apiKeyId,
+ autumnTrackInRequest: refundable,
+ },
+ { jobId: `alexandria-bill-${id}`, priority: 10 },
+ );
+ return true;
+ } catch (error) {
+ if (attempt < 3) {
+ await new Promise(resolve => setTimeout(resolve, 250 * attempt));
+ continue;
+ }
+ logger.error(
+ refundable
+ ? "Provider usage could not be queued for the ledger; refunding the Autumn charge and leaving the Exchange usage pending for reconciliation"
+ : "Provider usage could not be queued for the ledger; the firebill charge stands and the Exchange usage stays pending for reconciliation",
+ { chargeId: id, teamId, credits, error },
+ );
+ if (refundable)
+ await autumnService.refundCredits({
+ teamId,
+ value: credits,
+ featureId: featureIdForBillingEndpoint("scrape"),
+ properties: {
+ source: "alexandria",
+ endpoint: "scrape",
+ chargeId: id,
+ },
+ idempotencyKey: `fc:refund:scrape:${id}`,
+ });
+ return false;
+ }
+ }
+}