Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): route eligible scrapes through the Exchange (#4030)

Abimael Martell692d6b7e10fe
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
index 5c2d1cac3..1c4cf0407 100644
--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
@@ -6,12 +6,16 @@
describe("Audio format engine routing (buildFallbackList)", () => {
let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;
+ let clearExchangeProvidersForTest: typeof import("../../../lib/exchange.js").clearExchangeProvidersForTest;
+ let setExchangeProvidersForTest: typeof import("../../../lib/exchange.js").setExchangeProvidersForTest;
const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;
const originalIndexUrl = process.env.INDEX_DATABASE_URL;
+ const originalExchangeUrl = process.env.FIRE_EXCHANGE_URL;
beforeAll(async () => {
process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine";
+ process.env.FIRE_EXCHANGE_URL = "http://test-exchange";
process.env.INDEX_DATABASE_URL =
"postgresql://postgres:postgres@localhost:5432/postgres";
@@ -20,6 +24,14 @@ describe("Audio format engine routing (buildFallbackList)", () => {
({ buildFallbackList } = await import(
"../../../scraper/scrapeURL/engines/index.js"
));
+ ({
+ clearExchangeProvidersForTest,
+ setExchangeProvidersForTest,
+ } = await import("../../../lib/exchange.js"));
+ });
+
+ afterEach(() => {
+ clearExchangeProvidersForTest();
});
afterAll(() => {
@@ -33,6 +45,11 @@ describe("Audio format engine routing (buildFallbackList)", () => {
} else {
process.env.INDEX_DATABASE_URL = originalIndexUrl;
}
+ if (originalExchangeUrl === undefined) {
+ delete process.env.FIRE_EXCHANGE_URL;
+ } else {
+ process.env.FIRE_EXCHANGE_URL = originalExchangeUrl;
+ }
});
const buildStubMeta = (featureFlags: string[]) =>
@@ -110,4 +127,20 @@ describe("Audio format engine routing (buildFallbackList)", () => {
expect(engines).toContain("fire-engine;chrome-cdp");
});
+ it("does not route agent index-only requests through the Exchange", async () => {
+ setExchangeProvidersForTest([
+ { id: "acme", routes: [{ domains: ["profiles.example"] }] },
+ ]);
+
+ const meta = buildStubMeta([]);
+ meta.url = "https://profiles.example/person/example-person";
+ meta.options.formats = [{ type: "markdown" }];
+ meta.internalOptions.agentIndexOnly = true;
+ meta.internalOptions.teamFlags = { professionalProfileCompanyDataBeta: true };
+
+ const fallback = await buildFallbackList(meta);
+ const engines = fallback.map(f => f.engine);
+
+ expect(engines).toEqual(["index", "index;documents"]);
+ });
});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index a372c24e7..9168f7f60 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -165,6 +165,9 @@ const configSchema = z.object({
HIGHLIGHT_MODEL_TOKEN: z.string().optional(),
HIGHLIGHT_SHADOW_RATE: z.coerce.number().min(0).max(1).default(0),
+ // Exchange (routed data sources service)
+ FIRE_EXCHANGE_URL: z.url().optional(),
+
// Fire Engine
FIRE_ENGINE_BETA_URL: z.string().optional(),
FIRE_ENGINE_STAGING_URL: z.string().optional(),
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 650a4a0f9..a163c46fc 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -34,6 +34,7 @@ import { v2Router } from "./routes/v2";
import { nuqShutdown } from "./services/worker/nuq";
import { getErrorContactMessage } from "./lib/deployment";
import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";
+import { warmExchangeCatalog } from "./lib/exchange";
import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forcing";
import responseTime from "response-time";
import { shutdownWebhookQueue } from "./services/webhook";
@@ -132,6 +133,7 @@ async function startServer(port = DEFAULT_PORT) {
try {
await initializeBlocklist();
initializeEngineForcing();
+ warmExchangeCatalog();
} catch (error) {
logger.error("Failed to initialize API startup dependencies", {
error,
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 935ef612b..aaa59db8e 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -39,6 +39,8 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";
// TODO: figure out correct typing for this
const errorMap: Record<ErrorCodes, any> = {
+ // Terms responses are API-level, never transported through workers.
+ THIRD_PARTY_DATA_TERMS_REQUIRED: null,
SCRAPE_TIMEOUT: ScrapeJobTimeoutError,
MAP_TIMEOUT: MapTimeoutError,
UNKNOWN_ERROR: UnknownError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 62df2572b..d77a878f8 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -1,4 +1,5 @@
export type ErrorCodes =
+ | "THIRD_PARTY_DATA_TERMS_REQUIRED"
| "SCRAPE_TIMEOUT"
| "MAP_TIMEOUT"
| "UNKNOWN_ERROR"
diff --git a/apps/api/src/lib/exchange-request.ts b/apps/api/src/lib/exchange-request.ts
new file mode 100644
index 000000000..17238155b
--- /dev/null
+++ b/apps/api/src/lib/exchange-request.ts
@@ -0,0 +1,47 @@
+import {
+ getExchangeAccessForRequest,
+ type ExchangeAccess,
+} from "./exchange";
+
+type ExchangeFlags = Parameters<
+ typeof getExchangeAccessForRequest
+>[0]["flags"];
+
+/**
+ * Run the Exchange access gate against an API request body, sourcing the
+ * effective per-page options from a nested scrapeOptions object (crawl
+ * bodies) or the body itself (scrape and batch scrape bodies), with v1
+ * pageOptions fallbacks.
+ */
+export function getExchangeAccessForRequestBody(input: {
+ body: Record<string, any>;
+ flags: ExchangeFlags;
+ url: string;
+ zeroDataRetention: boolean;
+}): Promise<ExchangeAccess> {
+ const body = input.body ?? {};
+ const scrapeOptions =
+ typeof body.scrapeOptions === "object" && body.scrapeOptions !== null
+ ? body.scrapeOptions
+ : body;
+
+ return getExchangeAccessForRequest({
+ url: input.url,
+ formats: scrapeOptions.formats,
+ actions: scrapeOptions.actions,
+ headers: scrapeOptions.headers,
+ waitFor: scrapeOptions.waitFor,
+ mobile: scrapeOptions.mobile,
+ location: scrapeOptions.location,
+ proxy: scrapeOptions.proxy,
+ blockAds: scrapeOptions.blockAds,
+ profile: scrapeOptions.profile,
+ atsv: scrapeOptions.atsv ?? body.pageOptions?.atsv,
+ minAge: scrapeOptions.minAge,
+ includeTags: scrapeOptions.includeTags ?? body.pageOptions?.includeTags,
+ excludeTags: scrapeOptions.excludeTags ?? body.pageOptions?.excludeTags,
+ zeroDataRetention: input.zeroDataRetention,
+ lockdown: scrapeOptions.lockdown ?? body.lockdown,
+ flags: input.flags,
+ });
+}
diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.ts
new file mode 100644
index 000000000..416e2b627
--- /dev/null
+++ b/apps/api/src/lib/exchange.test.ts
@@ -0,0 +1,589 @@
+import { fetch } from "undici";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { config } from "../config";
+import {
+ canUseExchangeForRequest,
+ clearExchangeProvidersForTest,
+ reportExchangeBilling,
+ getExchangeAccessForRequest,
+ getExchangeRequestLogContext,
+ getExchangeResponseLogContext,
+ getExchangeSuccessCredits,
+ getThirdPartyDataTermsRequiredResponse,
+ isSuccessfulExchangeStatusCode,
+ isSupportedExchangeFormatRequest,
+ resolveExchangeProvider,
+ setExchangeProvidersForTest,
+} from "./exchange";
+
+vi.mock("undici", () => ({
+ fetch: vi.fn(),
+}));
+
+const originalConfig = {
+ FIRE_EXCHANGE_URL: config.FIRE_EXCHANGE_URL,
+ USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,
+};
+
+const ACME_TERMS = { key: "acme", version: "2026-01-01" };
+
+const TEST_PROVIDERS = [
+ {
+ id: "acme",
+ creditsCost: 12,
+ terms: ACME_TERMS,
+ routes: [
+ {
+ domains: ["profiles.example", "www.profiles.example"],
+ pathPrefixes: ["/person/", "/company/"],
+ },
+ ],
+ },
+ {
+ id: "openfacts",
+ creditsCost: 0,
+ routes: [{ domains: ["facts.example"] }],
+ },
+];
+
+const ENABLED_EXCHANGE_FLAGS = {
+ professionalProfileCompanyDataBeta: true,
+ organizationDataSourceAccess: {
+ acme: {
+ status: "enabled",
+ termsKey: "acme",
+ termsVersion: "2026-01-01",
+ termsAcceptedAt: "2026-01-01T00:00:00.000Z",
+ enabledAt: "2026-01-01T00:00:00.000Z",
+ disabledAt: null,
+ disabledReason: null,
+ },
+ },
+};
+
+describe("Exchange routing", () => {
+ beforeEach(() => {
+ vi.mocked(fetch).mockReset();
+ config.FIRE_EXCHANGE_URL = "https://exchange.example";
+ config.USE_DB_AUTHENTICATION = true;
+ setExchangeProvidersForTest(TEST_PROVIDERS);
+ });
+
+ afterEach(() => {
+ config.FIRE_EXCHANGE_URL = originalConfig.FIRE_EXCHANGE_URL;
+ config.USE_DB_AUTHENTICATION = originalConfig.USE_DB_AUTHENTICATION;
+ clearExchangeProvidersForTest();
+ });
+
+ it("resolves URLs to providers using catalog routes", async () => {
+ await expect(
+ resolveExchangeProvider(
+ "https://profiles.example/person/example-person/details/?trk=foo",
+ ),
+ ).resolves.toMatchObject({ id: "acme", creditsCost: 12 });
+ await expect(
+ resolveExchangeProvider("https://www.profiles.example/company/example"),
+ ).resolves.toMatchObject({ id: "acme" });
+ await expect(
+ resolveExchangeProvider("https://facts.example/any/path"),
+ ).resolves.toMatchObject({ id: "openfacts" });
+ await expect(
+ resolveExchangeProvider("https://profiles.example/jobs/example"),
+ ).resolves.toBeNull();
+ await expect(
+ resolveExchangeProvider("https://other.example/person/example"),
+ ).resolves.toBeNull();
+ await expect(resolveExchangeProvider("not a url")).resolves.toBeNull();
+
+ await expect(
+ resolveExchangeProvider("https://profiles.example/person/example-person"),
+ ).resolves.not.toBeNull();
+ });
+
+ it("respects path segment boundaries for prefixes without trailing slashes", async () => {
+ setExchangeProvidersForTest([
+ {
+ id: "acme",
+ creditsCost: 12,
+ routes: [{ domains: ["profiles.example"], pathPrefixes: ["/person"] }],
+ },
+ ]);
+
+ await expect(
+ resolveExchangeProvider("https://profiles.example/person"),
+ ).resolves.toMatchObject({ id: "acme" });
+ await expect(
+ resolveExchangeProvider("https://profiles.example/person/example"),
+ ).resolves.toMatchObject({ id: "acme" });
+ await expect(
+ resolveExchangeProvider("https://profiles.example/personality/details"),
+ ).resolves.toBeNull();
+ await expect(
+ resolveExchangeProvider("https://profiles.example/personnel"),
+ ).resolves.toBeNull();
+ });
+
+ it("serves the stale catalog while refreshing in the background", async () => {
+ setExchangeProvidersForTest(TEST_PROVIDERS, -1);
+ let resolveFetch: (value: unknown) => void = () => {};
+ vi.mocked(fetch).mockReturnValue(
+ new Promise(resolve => {
+ resolveFetch = resolve;
+ }) as ReturnType<typeof fetch>,
+ );
+
+ // The catalog is expired and the refresh never resolves during the
+ // request - the stale value must be served without waiting.
+ await expect(
+ resolveExchangeProvider("https://profiles.example/person/example"),
+ ).resolves.toMatchObject({ id: "acme" });
+ expect(fetch).toHaveBeenCalledTimes(1);
+
+ resolveFetch({ ok: false, status: 503 });
+ });
+
+ it("keeps serving the last good catalog when a refresh fails", async () => {
+ setExchangeProvidersForTest(TEST_PROVIDERS, -1);
+ vi.mocked(fetch).mockResolvedValue({
+ ok: false,
+ status: 503,
+ } as Awaited<ReturnType<typeof fetch>>);
+
+ await expect(
+ resolveExchangeProvider("https://profiles.example/person/example"),
+ ).resolves.toMatchObject({ id: "acme" });
+
+ // Let the background refresh settle - the failure must not clobber
+ // the catalog, and the failure TTL must throttle the next attempt.
+ await new Promise(resolve => setTimeout(resolve, 0));
+ await expect(
+ resolveExchangeProvider("https://profiles.example/person/example"),
+ ).resolves.toMatchObject({ id: "acme" });
+ expect(fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it("degrades to ineligible when the access check itself throws", async () => {
+ const explodingFlags = new Proxy(
+ {},
+ {
+ get() {
+ throw new Error("flags backend exploded");
+ },
+ },
+ ) as { professionalProfileCompanyDataBeta?: boolean };
+
+ await expect(
+ getExchangeAccessForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: explodingFlags,
+ }),
+ ).resolves.toEqual({ allowed: false, termsRequired: false });
+ });
+
+ it("caches failed provider catalog lookups briefly", async () => {
+ clearExchangeProvidersForTest();
+ vi.mocked(fetch).mockResolvedValue({
+ ok: false,
+ status: 503,
+ } as Awaited<ReturnType<typeof fetch>>);
+
+ await expect(
+ resolveExchangeProvider("https://profiles.example/person/example-person"),
+ ).resolves.toBeNull();
+ await expect(
+ resolveExchangeProvider("https://profiles.example/person/example-person"),
+ ).resolves.toBeNull();
+
+ expect(fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it("parses the provider catalog response", async () => {
+ clearExchangeProvidersForTest();
+ vi.mocked(fetch).mockResolvedValue({
+ ok: true,
+ status: 200,
+ json: async () => ({
+ success: true,
+ data: [
+ {
+ id: "acme",
+ name: "Acme",
+ description: "Structured records.",
+ creditsCost: 12,
+ terms: ACME_TERMS,
+ capabilities: {
+ scrape: {
+ resourceTypes: ["records"],
+ includes: [],
+ urlRoutes: [
+ {
+ domains: ["Profiles.Example"],
+ pathPrefixes: ["person/"],
+ },
+ ],
+ },
+ },
+ },
+ {
+ id: "searchonly",
+ name: "Search Only",
+ description: "No scrape routes.",
+ creditsCost: 0,
+ capabilities: { search: { modes: ["semantic"] } },
+ },
+ ],
+ }),
+ } as unknown as Awaited<ReturnType<typeof fetch>>);
+
+ await expect(
+ resolveExchangeProvider("https://profiles.example/person/example"),
+ ).resolves.toMatchObject({ id: "acme", terms: ACME_TERMS });
+ await expect(
+ resolveExchangeProvider("https://searchonly.example/anything"),
+ ).resolves.toBeNull();
+ expect(fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it("builds a compact request log context", () => {
+ expect(
+ getExchangeRequestLogContext(
+ "https://profiles.example/person/example-person/details/?trk=foo",
+ ),
+ ).toEqual({
+ url: "https://profiles.example/person/example-person/details/?trk=foo",
+ host: "profiles.example",
+ pathPrefix: "person",
+ });
+
+ expect(getExchangeRequestLogContext("not a url")).toBeUndefined();
+
+ // Embedded credentials must never reach the logs.
+ expect(
+ getExchangeRequestLogContext(
+ "https://user:[email protected]/person/example",
+ ),
+ ).toEqual({
+ url: "https://profiles.example/person/example",
+ host: "profiles.example",
+ pathPrefix: "person",
+ });
+ });
+
+ it("extracts response cache metadata for logs", () => {
+ expect(
+ getExchangeResponseLogContext({
+ cacheState: "hit",
+ cachedAt: "2026-06-21T10:00:00.000Z",
+ cacheAgeMs: 1000,
+ request_id: "req_123",
+ extra: "ignored",
+ }),
+ ).toEqual({
+ cacheState: "hit",
+ cachedAt: "2026-06-21T10:00:00.000Z",
+ cacheAgeMs: 1000,
+ providerRequestId: "req_123",
+ });
+
+ expect(getExchangeResponseLogContext(null)).toEqual({});
+ });
+
+ it("accepts only formats the Exchange can return directly", () => {
+ expect(isSupportedExchangeFormatRequest(undefined)).toBe(true);
+ expect(isSupportedExchangeFormatRequest([{ type: "markdown" }])).toBe(
+ true,
+ );
+ expect(isSupportedExchangeFormatRequest(["json"])).toBe(true);
+ expect(
+ isSupportedExchangeFormatRequest([
+ { type: "markdown" },
+ { type: "json" },
+ ]),
+ ).toBe(true);
+ expect(isSupportedExchangeFormatRequest([{ type: "html" }])).toBe(false);
+ // deterministicJson extractors run against page HTML, which Exchange
+ // responses do not carry.
+ expect(isSupportedExchangeFormatRequest([{ type: "deterministicJson" }])).toBe(
+ false,
+ );
+ expect(isSupportedExchangeFormatRequest([])).toBe(false);
+ });
+
+ it("allows eligible requests when access and terms are current", async () => {
+ await expect(
+ getExchangeAccessForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: ENABLED_EXCHANGE_FLAGS,
+ }),
+ ).resolves.toMatchObject({
+ allowed: true,
+ provider: { id: "acme", creditsCost: 12 },
+ });
+
+ await expect(
+ canUseExchangeForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "json" }],
+ actions: [{ type: "wait" }],
+ flags: ENABLED_EXCHANGE_FLAGS,
+ }),
+ ).resolves.toBe(false);
+
+ await expect(
+ canUseExchangeForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "json" }],
+ zeroDataRetention: true,
+ flags: ENABLED_EXCHANGE_FLAGS,
+ }),
+ ).resolves.toBe(false);
+
+ // Profile-backed scrapes expect session-specific content.
+ await expect(
+ canUseExchangeForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ profile: { id: "profile-1" },
+ flags: ENABLED_EXCHANGE_FLAGS,
+ }),
+ ).resolves.toBe(false);
+
+ // atsv requests stay on engines that support the flag.
+ await expect(
+ canUseExchangeForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ atsv: true,
+ flags: ENABLED_EXCHANGE_FLAGS,
+ }),
+ ).resolves.toBe(false);
+
+ // minAge asks for Firecrawl-cached data, which the Exchange never has.
+ await expect(
+ canUseExchangeForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ minAge: 3_600_000,
+ flags: ENABLED_EXCHANGE_FLAGS,
+ }),
+ ).resolves.toBe(false);
+
+ // Selector-based filtering does not apply to provider records.
+ await expect(
+ canUseExchangeForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ includeTags: ["article"],
+ flags: ENABLED_EXCHANGE_FLAGS,
+ }),
+ ).resolves.toBe(false);
+ await expect(
+ canUseExchangeForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ excludeTags: ["nav"],
+ flags: ENABLED_EXCHANGE_FLAGS,
+ }),
+ ).resolves.toBe(false);
+ });
+
+ it("requires the provider's terms before routing", async () => {
+ const access = await getExchangeAccessForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: { professionalProfileCompanyDataBeta: true },
+ });
+
+ expect(access).toEqual({
+ allowed: false,
+ termsRequired: true,
+ terms: ACME_TERMS,
+ });
+
+ expect(getThirdPartyDataTermsRequiredResponse(ACME_TERMS)).toMatchObject({
+ success: false,
+ code: "THIRD_PARTY_DATA_TERMS_REQUIRED",
+ requiresAction: {
+ type: "accept_terms",
+ terms: "acme",
+ version: "2026-01-01",
+ },
+ });
+ });
+
+ it("requires current terms when the accepted version is stale", async () => {
+ await expect(
+ getExchangeAccessForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: {
+ professionalProfileCompanyDataBeta: true,
+ organizationDataSourceAccess: {
+ acme: {
+ status: "enabled",
+ termsKey: "acme",
+ termsVersion: "2025-12-01",
+ },
+ },
+ },
+ }),
+ ).resolves.toEqual({
+ allowed: false,
+ termsRequired: true,
+ terms: ACME_TERMS,
+ });
+ });
+
+ it("does not route or prompt for terms when access is disabled", async () => {
+ await expect(
+ getExchangeAccessForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: {
+ professionalProfileCompanyDataBeta: true,
+ organizationDataSourceAccess: {
+ acme: {
+ status: "disabled",
+ termsKey: "acme",
+ termsVersion: "2026-01-01",
+ disabledAt: "2026-01-02T00:00:00.000Z",
+ disabledReason: "customer_disabled",
+ },
+ },
+ },
+ }),
+ ).resolves.toEqual({ allowed: false, termsRequired: false });
+ });
+
+ it("allows providers with no declared terms without acceptance", async () => {
+ await expect(
+ getExchangeAccessForRequest({
+ url: "https://facts.example/records/1",
+ formats: [{ type: "markdown" }],
+ flags: { professionalProfileCompanyDataBeta: true },
+ }),
+ ).resolves.toMatchObject({ allowed: true, provider: { id: "openfacts" } });
+
+ await expect(
+ getExchangeAccessForRequest({
+ url: "https://facts.example/records/1",
+ formats: [{ type: "markdown" }],
+ flags: {
+ professionalProfileCompanyDataBeta: true,
+ organizationDataSourceAccess: {
+ openfacts: { status: "disabled" },
+ },
+ },
+ }),
+ ).resolves.toEqual({ allowed: false, termsRequired: false });
+ });
+
+ it("does not route unless the beta flag is enabled", async () => {
+ await expect(
+ canUseExchangeForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ }),
+ ).resolves.toBe(false);
+
+ await expect(
+ canUseExchangeForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: { professionalProfileCompanyDataBeta: false },
+ }),
+ ).resolves.toBe(false);
+ });
+
+ it("does not route unless the Exchange is configured", async () => {
+ config.FIRE_EXCHANGE_URL = undefined;
+
+ await expect(
+ canUseExchangeForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: ENABLED_EXCHANGE_FLAGS,
+ }),
+ ).resolves.toBe(false);
+ });
+
+ it("bills the reported credit cost only for successful handled responses", () => {
+ expect(isSuccessfulExchangeStatusCode(200)).toBe(true);
+ expect(isSuccessfulExchangeStatusCode(204)).toBe(true);
+ expect(isSuccessfulExchangeStatusCode(304)).toBe(true);
+ expect(isSuccessfulExchangeStatusCode(404)).toBe(false);
+
+ expect(
+ getExchangeSuccessCredits({
+ exchange: { handled: true, creditsCost: 12 },
+ statusCode: 200,
+ }),
+ ).toBe(12);
+
+ expect(
+ getExchangeSuccessCredits({
+ exchange: { handled: true, creditsCost: 0 },
+ statusCode: 304,
+ }),
+ ).toBe(0);
+
+ expect(
+ getExchangeSuccessCredits({
+ exchange: { handled: true, creditsCost: 12 },
+ statusCode: 404,
+ }),
+ ).toBeNull();
+
+ expect(
+ getExchangeSuccessCredits({
+ statusCode: 200,
+ }),
+ ).toBeNull();
+ });
+
+ it("reports billing outcomes without throwing on service failures", async () => {
+ vi.mocked(fetch).mockResolvedValue({
+ ok: true,
+ status: 200,
+ } as Awaited<ReturnType<typeof fetch>>);
+
+ await reportExchangeBilling({
+ accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",
+ status: "confirmed",
+ billingReference: "bill-1",
+ });
+
+ expect(fetch).toHaveBeenCalledWith(
+ "https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing",
+ expect.objectContaining({
+ method: "POST",
+ body: JSON.stringify({ status: "confirmed", billingReference: "bill-1" }),
+ }),
+ );
+
+ await reportExchangeBilling({
+ accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",
+ status: "void",
+ });
+
+ expect(fetch).toHaveBeenLastCalledWith(
+ "https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing",
+ expect.objectContaining({
+ method: "POST",
+ body: JSON.stringify({ status: "void" }),
+ }),
+ );
+
+ vi.mocked(fetch).mockRejectedValue(new Error("connect timeout"));
+ await expect(
+ reportExchangeBilling({
+ accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",
+ status: "confirmed",
+ billingReference: "bill-1",
+ }),
+ ).resolves.toBeUndefined();
+ });
+});
diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.ts
new file mode 100644
index 000000000..9de05a473
--- /dev/null
+++ b/apps/api/src/lib/exchange.ts
@@ -0,0 +1,639 @@
+import { fetch } from "undici";
+import { z } from "zod";
+
+import { config } from "../config";
+import type { FormatObject } from "../controllers/v2/types";
+import { logger as rootLogger } from "./logger";
+
+type OrganizationDataSourceAccessRecord = {
+ status?: string | null;
+ termsKey?: string | null;
+ termsVersion?: string | null;
+ termsAcceptedAt?: string | null;
+ enabledAt?: string | null;
+ disabledAt?: string | null;
+ disabledReason?: string | null;
+};
+
+type OrganizationDataSourceAccess = Record<
+ string,
+ OrganizationDataSourceAccessRecord | null | undefined
+>;
+
+type RouteInput = {
+ url: string;
+ formats?: FormatObject[] | unknown[];
+ actions?: unknown[];
+ headers?: Record<string, unknown>;
+ waitFor?: number;
+ mobile?: boolean;
+ location?: unknown;
+ proxy?: unknown;
+ blockAds?: boolean;
+ profile?: unknown;
+ atsv?: boolean;
+ minAge?: number;
+ includeTags?: unknown[];
+ excludeTags?: unknown[];
+ zeroDataRetention?: boolean;
+ lockdown?: boolean;
+ flags?: {
+ professionalProfileCompanyDataBeta?: boolean;
+ organizationDataSourceAccess?: OrganizationDataSourceAccess | null;
+ } | null;
+};
+
+export type ExchangeScrapeMetadata = {
+ handled: true;
+ creditsCost: number;
+ accessEventId?: string;
+ integrationId?: string;
+};
+
+export type ExchangeTerms = {
+ key: string;
+ version: string;
+};
+
+export type ExchangeProvider = {
+ id: string;
+ creditsCost: number;
+ terms?: ExchangeTerms;
+ routes: {
+ domains: Set<string>;
+ pathPrefixes: string[];
+ }[];
+};
+
+// deterministicJson is deliberately unsupported: its extractor scripts run
+// against page HTML, which Exchange responses do not carry.
+const SUPPORTED_FORMATS = new Set(["markdown", "json"]);
+const EXCHANGE_BETA_FLAG = "professionalProfileCompanyDataBeta";
+const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED";
+const THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE =
+ "An organization admin must accept this data source's terms before this URL can be processed.";
+
+const EXCHANGE_PROVIDERS_PATH = "/v1/providers";
+const EXCHANGE_PROVIDERS_TIMEOUT_MS = 2_000;
+const EXCHANGE_PROVIDERS_TTL_MS = 60_000;
+const EXCHANGE_PROVIDERS_FAILURE_TTL_MS = 30_000;
+
+const exchangeProvidersSchema = z.object({
+ success: z.literal(true),
+ data: z.array(
+ z
+ .object({
+ id: z.string(),
+ // No .catch() here: a malformed credit cost must reject the catalog
+ // (keeping the last good one) rather than silently billing 0.
+ creditsCost: z.number().int().nonnegative(),
+ terms: z
+ .object({
+ key: z.string(),
+ version: z.string(),
+ })
+ .optional(),
+ capabilities: z
+ .object({
+ scrape: z
+ .object({
+ urlRoutes: z
+ .array(
+ z
+ .object({
+ domains: z.string().array(),
+ pathPrefixes: z.string().array(),
+ })
+ .passthrough(),
+ )
+ .optional(),
+ })
+ .passthrough()
+ .optional(),
+ })
+ .passthrough(),
+ })
+ .passthrough(),
+ ),
+});
+
+let cachedProviders:
+ | {
+ expiresAt: number;
+ value: ExchangeProvider[] | null;
+ }
+ | undefined;
+let providersRequest: Promise<ExchangeProvider[] | null> | undefined;
+
+function normalizeHost(host: string): string {
+ return host.trim().toLowerCase().replace(/\.$/, "");
+}
+
+function normalizePathPrefix(prefix: string): string {
+ const trimmed = prefix.trim();
+ if (!trimmed) {
+ return "/";
+ }
+
+ return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
+}
+
+function getExchangeBaseUrl(): string | null {
+ if (!config.FIRE_EXCHANGE_URL) {
+ return null;
+ }
+
+ return config.FIRE_EXCHANGE_URL.replace(/\/+$/, "");
+}
+
+function normalizeProviders(
+ raw: z.infer<typeof exchangeProvidersSchema>,
+): ExchangeProvider[] {
+ return raw.data
+ .map(provider => ({
+ id: provider.id,
+ creditsCost: provider.creditsCost,
+ ...(provider.terms === undefined ? {} : { terms: provider.terms }),
+ routes: (provider.capabilities.scrape?.urlRoutes ?? []).map(route => ({
+ domains: new Set(route.domains.map(normalizeHost)),
+ pathPrefixes: route.pathPrefixes.map(normalizePathPrefix),
+ })),
+ }))
+ .filter(provider => provider.routes.length > 0);
+}
+
+async function fetchExchangeProviders(): Promise<ExchangeProvider[] | null> {
+ const baseUrl = getExchangeBaseUrl();
+ if (!baseUrl) {
+ return null;
+ }
+
+ try {
+ const response = await fetch(`${baseUrl}${EXCHANGE_PROVIDERS_PATH}`, {
+ method: "GET",
+ signal: AbortSignal.timeout(EXCHANGE_PROVIDERS_TIMEOUT_MS),
+ });
+
+ if (!response.ok) {
+ rootLogger.warn("Exchange providers request failed", {
+ statusCode: response.status,
+ });
+ return null;
+ }
+
+ const parsed = exchangeProvidersSchema.parse(await response.json());
+ return normalizeProviders(parsed);
+ } catch (error) {
+ rootLogger.warn("Exchange providers request errored", { error });
+ return null;
+ }
+}
+
+async function getExchangeProviders(): Promise<ExchangeProvider[] | null> {
+ if (cachedProviders && cachedProviders.expiresAt > Date.now()) {
+ return cachedProviders.value;
+ }
+
+ if (!providersRequest) {
+ providersRequest = fetchExchangeProviders()
+ .then(providers => {
+ if (providers === null) {
+ // Keep serving the last good catalog through transient outages;
+ // the failure TTL only delays the next refresh attempt.
+ cachedProviders = {
+ value: cachedProviders?.value ?? null,
+ expiresAt: Date.now() + EXCHANGE_PROVIDERS_FAILURE_TTL_MS,
+ };
+ } else {
+ cachedProviders = {
+ value: providers,
+ expiresAt: Date.now() + EXCHANGE_PROVIDERS_TTL_MS,
+ };
+ }
+ return cachedProviders.value;
+ })
+ .finally(() => {
+ providersRequest = undefined;
+ });
+ }
+
+ // Serve the stale catalog while the refresh runs in the background so
+ // request latency never depends on the catalog endpoint; only the very
+ // first lookup after boot has nothing to serve and waits.
+ if (cachedProviders) {
+ return cachedProviders.value;
+ }
+
+ return providersRequest;
+}
+
+function providerMatchesUrl(provider: ExchangeProvider, inputUrl: string): boolean {
+ let parsed: URL;
+ try {
+ parsed = new URL(inputUrl);
+ } catch {
+ return false;
+ }
+
+ const host = normalizeHost(parsed.hostname);
+ const pathname = parsed.pathname || "/";
+
+ return provider.routes.some(route => {
+ if (!route.domains.has(host)) {
+ return false;
+ }
+
+ if (route.pathPrefixes.length === 0) {
+ return true;
+ }
+
+ // Prefix matches respect path segment boundaries: "/person" matches
+ // "/person" and "/person/x" but never "/personality".
+ return route.pathPrefixes.some(prefix =>
+ prefix.endsWith("/")
+ ? pathname.startsWith(prefix)
+ : pathname === prefix || pathname.startsWith(`${prefix}/`),
+ );
+ });
+}
+
+export async function resolveExchangeProvider(
+ inputUrl: string,
+): Promise<ExchangeProvider | null> {
+ const providers = await getExchangeProviders();
+ if (providers === null) {
+ return null;
+ }
+
+ return providers.find(provider => providerMatchesUrl(provider, inputUrl)) ?? null;
+}
+
+export function getExchangeRequestLogContext(inputUrl: string):
+ | {
+ url: string;
+ host: string;
+ pathPrefix: string | null;
+ }
+ | undefined {
+ let parsed: URL;
+ try {
+ parsed = new URL(inputUrl);
+ } catch {
+ return undefined;
+ }
+
+ // Never log embedded credentials from user-submitted URLs.
+ parsed.username = "";
+ parsed.password = "";
+
+ return {
+ url: parsed.href,
+ host: parsed.hostname.toLowerCase(),
+ pathPrefix:
+ parsed.pathname
+ .split("/")
+ .map(part => part.trim())
+ .filter(part => part.length > 0)[0] ?? null,
+ };
+}
+
+export function getExchangeResponseLogContext(meta: unknown): {
+ cacheState?: string;
+ cachedAt?: string;
+ cacheAgeMs?: number;
+ providerRequestId?: string;
+} {
+ if (typeof meta !== "object" || meta === null) {
+ return {};
+ }
+
+ const record = meta as Record<string, unknown>;
+ const requestId = record.request_id ?? record.requestId;
+
+ return {
+ ...(typeof record.cacheState === "string"
+ ? { cacheState: record.cacheState }
+ : {}),
+ ...(typeof record.cachedAt === "string"
+ ? { cachedAt: record.cachedAt }
+ : {}),
+ ...(typeof record.cacheAgeMs === "number"
+ ? { cacheAgeMs: record.cacheAgeMs }
+ : {}),
+ ...(typeof requestId === "string" ? { providerRequestId: requestId } : {}),
+ };
+}
+
+export function isSuccessfulExchangeStatusCode(statusCode: number): boolean {
+ return (statusCode >= 200 && statusCode < 300) || statusCode === 304;
+}
+
+export function isSupportedExchangeFormatRequest(
+ formats?: FormatObject[] | unknown[],
+): boolean {
+ if (formats === undefined) {
+ return true;
+ }
+
+ if (!Array.isArray(formats) || formats.length === 0) {
+ return false;
+ }
+
+ return formats.every(format => {
+ const type =
+ typeof format === "string"
+ ? format
+ : typeof format === "object" && format !== null && "type" in format
+ ? (format as { type?: unknown }).type
+ : undefined;
+
+ return typeof type === "string" && SUPPORTED_FORMATS.has(type);
+ });
+}
+
+type DataSourceAccessDecision = "allowed" | "terms_required" | "not_enabled";
+
+function getProviderAccessDecision(
+ provider: ExchangeProvider,
+ flags: RouteInput["flags"],
+): DataSourceAccessDecision {
+ if (config.USE_DB_AUTHENTICATION !== true) {
+ return "allowed";
+ }
+
+ const access = flags?.organizationDataSourceAccess?.[provider.id];
+ const entry = typeof access === "object" && access !== null ? access : null;
+
+ if (provider.terms === undefined) {
+ return entry !== null && entry.status !== "enabled" ? "not_enabled" : "allowed";
+ }
+
+ if (entry === null) {
+ return "terms_required";
+ }
+
+ if (entry.status !== "enabled") {
+ return "not_enabled";
+ }
+
+ return entry.termsKey === provider.terms.key &&
+ entry.termsVersion === provider.terms.version
+ ? "allowed"
+ : "terms_required";
+}
+
+function isExchangeEligibleRequest(input: RouteInput): boolean {
+ if (input.flags?.[EXCHANGE_BETA_FLAG] !== true) {
+ return false;
+ }
+
+ if (!config.FIRE_EXCHANGE_URL) {
+ return false;
+ }
+
+ if (!input.url) {
+ return false;
+ }
+
+ if (input.zeroDataRetention || input.lockdown) {
+ return false;
+ }
+
+ if (Array.isArray(input.actions) && input.actions.length > 0) {
+ return false;
+ }
+
+ if (input.headers && Object.keys(input.headers).length > 0) {
+ return false;
+ }
+
+ if (input.waitFor !== undefined && input.waitFor !== 0) {
+ return false;
+ }
+
+ if (input.mobile || input.location || input.blockAds === false) {
+ return false;
+ }
+
+ // Profile-backed scrapes expect session-specific content, which the
+ // Exchange cannot serve.
+ if (input.profile !== undefined) {
+ return false;
+ }
+
+ // atsv is only supported by browser engines; requests that set it keep an
+ // engine that can honor it instead of routing to the Exchange.
+ if (input.atsv === true) {
+ return false;
+ }
+
+ // minAge requests ask for Firecrawl-cached data; the Exchange serves
+ // provider data and Firecrawl never caches it, so the semantics cannot
+ // be honored here.
+ if (input.minAge !== undefined) {
+ return false;
+ }
+
+ // Selector-based content filtering does not apply to provider records.
+ if (
+ (Array.isArray(input.includeTags) && input.includeTags.length > 0) ||
+ (Array.isArray(input.excludeTags) && input.excludeTags.length > 0)
+ ) {
+ return false;
+ }
+
+ if (input.proxy === "stealth" || input.proxy === "enhanced") {
+ return false;
+ }
+
+ if (!isSupportedExchangeFormatRequest(input.formats)) {
+ return false;
+ }
+
+ return true;
+}
+
+export type ExchangeAccess =
+ | {
+ allowed: true;
+ termsRequired: false;
+ provider: ExchangeProvider;
+ }
+ | {
+ allowed: false;
+ termsRequired: true;
+ terms: ExchangeTerms;
+ }
+ | {
+ allowed: false;
+ termsRequired: false;
+ };
+
+export async function getExchangeAccessForRequest(
+ input: RouteInput,
+): Promise<ExchangeAccess> {
+ // The Exchange gate sits on the hot path of every scrape request; no
+ // failure inside it may ever fail the request itself. Anything unexpected
+ // degrades to "not eligible" and the request continues on the normal path.
+ try {
+ if (!isExchangeEligibleRequest(input)) {
+ return { allowed: false, termsRequired: false };
+ }
+
+ const provider = await resolveExchangeProvider(input.url);
+ if (provider === null) {
+ return { allowed: false, termsRequired: false };
+ }
+
+ const decision = getProviderAccessDecision(provider, input.flags);
+ if (decision === "terms_required" && provider.terms !== undefined) {
+ return { allowed: false, termsRequired: true, terms: provider.terms };
+ }
+ if (decision !== "allowed") {
+ return { allowed: false, termsRequired: false };
+ }
+
+ return { allowed: true, termsRequired: false, provider };
+ } catch (error) {
+ rootLogger.warn("Exchange access check errored; treating as ineligible", {
+ error,
+ });
+ return { allowed: false, termsRequired: false };
+ }
+}
+
+export async function canUseExchangeForRequest(
+ input: RouteInput,
+): Promise<boolean> {
+ return (await getExchangeAccessForRequest(input)).allowed;
+}
+
+function getThirdPartyDataTermsSettingsUrl(): string {
+ return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/settings?tab=data-sources`;
+}
+
+export function getThirdPartyDataTermsRequiredResponse(terms: ExchangeTerms) {
+ return {
+ success: false as const,
+ code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE as "THIRD_PARTY_DATA_TERMS_REQUIRED",
+ error: THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE,
+ requiresAction: {
+ type: "accept_terms",
+ terms: terms.key,
+ version: terms.version,
+ url: getThirdPartyDataTermsSettingsUrl(),
+ },
+ };
+}
+
+export function getExchangeSuccessCredits(input: {
+ exchange?: ExchangeScrapeMetadata;
+ statusCode?: number | null;
+}): number | null {
+ if (input.exchange?.handled !== true) {
+ return null;
+ }
+
+ const statusCode = input.statusCode;
+ if (
+ statusCode === undefined ||
+ statusCode === null ||
+ !isSuccessfulExchangeStatusCode(statusCode)
+ ) {
+ return null;
+ }
+
+ return input.exchange.creditsCost;
+}
+
+const EXCHANGE_BILLING_TIMEOUT_MS = 5_000;
+
+/**
+ * Report the billing outcome of a delivered Exchange access so the service
+ * can reconcile its ledger: "confirmed" once the customer was billed, "void"
+ * when the delivered access was ultimately discarded and never billed.
+ * Failures only log - the service flags unresolved events for follow-up.
+ */
+export async function reportExchangeBilling(input: {
+ accessEventId: string;
+ status: "confirmed" | "void";
+ billingReference?: string;
+}): Promise<void> {
+ const baseUrl = getExchangeBaseUrl();
+ if (!baseUrl) {
+ return;
+ }
+
+ try {
+ const response = await fetch(
+ `${baseUrl}/v1/access-events/${encodeURIComponent(input.accessEventId)}/billing`,
+ {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ status: input.status,
+ ...(input.billingReference === undefined
+ ? {}
+ : { billingReference: input.billingReference }),
+ }),
+ signal: AbortSignal.timeout(EXCHANGE_BILLING_TIMEOUT_MS),
+ },
+ );
+
+ if (!response.ok) {
+ rootLogger.warn("Exchange billing report failed", {
+ accessEventId: input.accessEventId,
+ status: input.status,
+ statusCode: response.status,
+ });
+ }
+ } catch (error) {
+ rootLogger.warn("Exchange billing report errored", {
+ accessEventId: input.accessEventId,
+ status: input.status,
+ error,
+ });
+ }
+}
+
+/**
+ * Warm the provider catalog at process startup so the first flagged-org
+ * request never waits on the fetch; after this, stale-while-revalidate
+ * keeps every lookup in-memory. No-op when the Exchange is not configured;
+ * never throws.
+ */
+export function warmExchangeCatalog(): void {
+ if (!config.FIRE_EXCHANGE_URL) {
+ return;
+ }
+
+ void getExchangeProviders();
+}
+
+export function setExchangeProvidersForTest(
+ providers: {
+ id: string;
+ creditsCost?: number;
+ terms?: ExchangeTerms;
+ routes: { domains: string[]; pathPrefixes?: string[] }[];
+ }[],
+ ttlMs = 300_000,
+) {
+ cachedProviders = {
+ value: providers.map(provider => ({
+ id: provider.id,
+ creditsCost: provider.creditsCost ?? 0,
+ ...(provider.terms === undefined ? {} : { terms: provider.terms }),
+ routes: provider.routes.map(route => ({
+ domains: new Set(route.domains.map(normalizeHost)),
+ pathPrefixes: (route.pathPrefixes ?? []).map(normalizePathPrefix),
+ })),
+ })),
+ expiresAt: Date.now() + ttlMs,
+ };
+}
+
+export function clearExchangeProvidersForTest() {
+ cachedProviders = undefined;
+ providersRequest = undefined;
+}
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts
index a6c67caee..9c1f9e768 100644
--- a/apps/api/src/lib/scrape-billing.test.ts
+++ b/apps/api/src/lib/scrape-billing.test.ts
@@ -6,6 +6,33 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";
import type { ThreatDecision } from "./threat-protection/types";
describe("calculateCreditsToBeBilled", () => {
+ it("bills handled Exchange successes at the reported credit cost", async () => {
+ const credits = await calculateCreditsToBeBilled(
+ {
+ formats: [{ type: "markdown" }],
+ } as any,
+ {
+ teamId: "team-id",
+ },
+ {
+ metadata: {
+ statusCode: 200,
+ url: "https://profiles.example/person/example-person",
+ proxyUsed: "basic",
+ },
+ } as any,
+ {
+ totalCost: 0,
+ } as any,
+ {} as any,
+ undefined,
+ undefined,
+ { handled: true, creditsCost: 12 },
+ );
+
+ expect(credits).toBe(12);
+ });
+
it("bills X/Twitter scrapes at 30 credits", async () => {
const credits = await calculateCreditsToBeBilled(
{
@@ -142,6 +169,7 @@ const billWithDecisions = (args: {
{} as any,
args.error,
undefined,
+ undefined,
args.threatDecisions,
);
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index b393646d7..58347aec3 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -10,6 +10,10 @@ import { hasFormatOfType } from "./format-utils";
import { TransportableError } from "./error";
import { FeatureFlag } from "../scraper/scrapeURL/engines";
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
+import {
+ ExchangeScrapeMetadata,
+ getExchangeSuccessCredits,
+} from "./exchange";
import type { ThreatDecision } from "./threat-protection/types";
import { UnsafeDomainBlockedError } from "./threat-protection/error";
@@ -64,6 +68,7 @@ export async function calculateCreditsToBeBilled(
flags: TeamFlags,
error?: Error | null,
unsupportedFeatures?: Set<FeatureFlag>,
+ exchange?: ExchangeScrapeMetadata,
// Threat protection decisions for this scrape (initial + redirect checks,
// in order). Each decision with `providerConsulted` bills a scan fee (+2
// per unique scanned URL) on top of the scrape's own cost — on both success
@@ -112,6 +117,14 @@ export async function calculateCreditsToBeBilled(
return creditsToBeBilled + threatScanCredits;
}
+ const exchangeCredits = getExchangeSuccessCredits({
+ exchange,
+ statusCode: document.metadata?.statusCode,
+ });
+ if (exchangeCredits !== null) {
+ return exchangeCredits + threatScanCredits;
+ }
+
let creditsToBeBilled = 1; // Assuming 1 credit per document
if (options.lockdown) {
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index 5528a9aa8..73f553e3b 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -29,6 +29,9 @@ import {
CREDITS_FEATURE_ID,
} from "../services/autumn/autumn.service";
import { getTeamBalance } from "../services/autumn/usage";
+import { getThirdPartyDataTermsRequiredResponse } from "../lib/exchange";
+import { getExchangeAccessForRequestBody } from "../lib/exchange-request";
+import { getScrapeZDR } from "../lib/zdr-helpers";
export function checkCreditsMiddleware(
_minimum?: number,
@@ -284,10 +287,58 @@ export function blocklistMiddleware(
req: RequestWithMaybeACUC<any, any, any>,
res: Response,
next: NextFunction,
+) {
+ return blocklistGate(req, res, next, { exchange: false });
+}
+
+/**
+ * Blocklist gate for single-URL scrape-shaped routes (scrape, crawl), where
+ * an Exchange-eligible URL may bypass the blocklist because the exchange
+ * engine can serve it. Everything else (map, search, batch scrape, monitors)
+ * keeps plain blocklist behavior - batch stays out until its jobs carry the
+ * access flags the worker-side recheck needs.
+ */
+export function scrapeBlocklistMiddleware(
+ req: RequestWithMaybeACUC<any, any, any>,
+ res: Response,
+ next: NextFunction,
+) {
+ return blocklistGate(req, res, next, { exchange: true });
+}
+
+function blocklistGate(
+ req: RequestWithMaybeACUC<any, any, any>,
+ res: Response,
+ next: NextFunction,
+ options: { exchange: boolean },
) {
(async () => {
+ const zeroDataRetention =
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ req.body?.zeroDataRetention === true;
+ const exchangeAccess =
+ options.exchange &&
+ typeof req.body.url === "string" &&
+ (await getExchangeAccessForRequestBody({
+ body: req.body,
+ flags: req.acuc?.flags ?? null,
+ url: req.body.url,
+ zeroDataRetention,
+ }));
+ const canUseExchange =
+ typeof exchangeAccess === "object" && exchangeAccess.allowed;
+
+ if (typeof exchangeAccess === "object" && exchangeAccess.termsRequired) {
+ if (!res.headersSent) {
+ return res
+ .status(403)
+ .json(getThirdPartyDataTermsRequiredResponse(exchangeAccess.terms));
+ }
+ }
+
if (
typeof req.body.url === "string" &&
+ !canUseExchange &&
isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {
team_id: req.acuc?.team_id ?? null,
origin: typeof req.body.origin === "string" ? req.body.origin : null,
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index bc92811d1..d2cdf6dff 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -28,6 +28,7 @@ import {
authMiddleware,
checkCreditsMiddleware,
blocklistMiddleware,
+ scrapeBlocklistMiddleware,
countryCheck,
idempotencyMiddleware,
requestTimingMiddleware,
@@ -51,7 +52,7 @@ v1Router.post(
authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),
countryCheck,
checkCreditsMiddleware(1),
- blocklistMiddleware,
+ scrapeBlocklistMiddleware,
wrap(scrapeController),
);
@@ -60,7 +61,7 @@ v1Router.post(
authMiddleware(RateLimiterMode.Crawl),
countryCheck,
checkCreditsMiddleware(),
- blocklistMiddleware,
+ scrapeBlocklistMiddleware,
idempotencyMiddleware,
wrap(crawlController),
);
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index e3a0991dd..31c9b3997 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -38,6 +38,7 @@ import {
authMiddleware,
checkCreditsMiddleware,
blocklistMiddleware,
+ scrapeBlocklistMiddleware,
countryCheck,
idempotencyMiddleware,
requestTimingMiddleware,
@@ -210,7 +211,7 @@ v2Router.post(
authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),
countryCheck,
checkCreditsMiddleware(1),
- blocklistMiddleware,
+ scrapeBlocklistMiddleware,
wrap(scrapeController),
);
@@ -257,7 +258,7 @@ v2Router.post(
authMiddleware(RateLimiterMode.Crawl),
countryCheck,
checkCreditsMiddleware(),
- blocklistMiddleware,
+ scrapeBlocklistMiddleware,
idempotencyMiddleware,
wrap(crawlController),
);
diff --git a/apps/api/src/scraper/scrapeURL/engines/exchange.ts b/apps/api/src/scraper/scrapeURL/engines/exchange.ts
new file mode 100644
index 000000000..c6c41de2d
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/exchange.ts
@@ -0,0 +1,186 @@
+import { z } from "zod";
+
+import { Meta } from "..";
+import { EngineScrapeResult } from ".";
+import { config } from "../../../config";
+import {
+ getExchangeRequestLogContext,
+ getExchangeResponseLogContext,
+} from "../../../lib/exchange";
+import { setSpanAttributes, withSpan } from "../../../lib/otel-tracer";
+import { robustFetch } from "../lib/fetch";
+import { EngineError } from "../error";
+
+const exchangeScrapeResponseSchema = z.union([
+ z
+ .object({
+ success: z.literal(true),
+ accessEventId: z.string().optional(),
+ // No .catch() here: a malformed credit cost must fail the scrape
+ // loudly rather than silently billing 0 for a delivered access.
+ creditsCost: z.number().int().nonnegative(),
+ data: z
+ .object({
+ url: z.string().optional(),
+ title: z.string().optional(),
+ description: z.string().optional(),
+ source: z
+ .object({
+ provider: z.string().optional(),
+ })
+ .passthrough()
+ .optional(),
+ metadata: z.record(z.string(), z.unknown()).optional(),
+ markdown: z.string().optional(),
+ json: z.unknown().optional(),
+ })
+ .passthrough(),
+ })
+ .passthrough(),
+ z
+ .object({
+ success: z.literal(false),
+ error: z
+ .object({
+ code: z.string().optional(),
+ message: z.string().optional(),
+ })
+ .passthrough()
+ .optional(),
+ })
+ .passthrough(),
+]);
+
+export function exchangeMaxReasonableTime(meta: Meta): number {
+ return meta.options.timeout ?? 60_000;
+}
+
+function escapeHtml(value: string): string {
+ return value
+ .replace(/&/g, "&amp;")
+ .replace(/</g, "&lt;")
+ .replace(/>/g, "&gt;")
+ .replace(/"/g, "&quot;");
+}
+
+// Exchange responses carry no page HTML; synthesize a minimal head so the
+// metadata transformer can populate the document's title and description.
+function buildMetadataHtml(title?: string, description?: string): string {
+ const titleTag = title === undefined ? "" : `<title>${escapeHtml(title)}</title>`;
+ const descriptionTag =
+ description === undefined
+ ? ""
+ : `<meta name="description" content="${escapeHtml(description)}">`;
+ return `<!DOCTYPE html><html><head>${titleTag}${descriptionTag}</head><body></body></html>`;
+}
+
+export async function scrapeURLWithExchange(
+ meta: Meta,
+): Promise<EngineScrapeResult> {
+ return withSpan("engine.exchange.scrape", async span => {
+ const startTime = Date.now();
+ const url = meta.rewrittenUrl ?? meta.url;
+ const requestLogContext = getExchangeRequestLogContext(url);
+ const logger = meta.logger.child({ method: "scrapeURLWithExchange" });
+
+ setSpanAttributes(span, {
+ "engine.type": "exchange",
+ // Follow the same credential redaction as the log context.
+ "engine.url": requestLogContext?.url ?? "",
+ "engine.team_id": meta.internalOptions.teamId,
+ });
+
+ logger.info("Exchange scrape started", {
+ ...requestLogContext,
+ scrapeId: meta.id,
+ teamId: meta.internalOptions.teamId,
+ maxAge: meta.options.maxAge,
+ });
+
+ try {
+ const response = await robustFetch({
+ url: `${config.FIRE_EXCHANGE_URL!.replace(/\/+$/, "")}/v1/scrape`,
+ method: "POST",
+ body: {
+ requestId: meta.id,
+ teamId: meta.internalOptions.teamId,
+ url,
+ formats: ["markdown", "json"],
+ ...(meta.options.maxAge === undefined
+ ? {}
+ : { maxAge: meta.options.maxAge }),
+ },
+ logger: logger.child({ method: "exchangeScrape/robustFetch" }),
+ tryCount: 2,
+ ignoreFailureStatus: true,
+ mock: meta.mock,
+ abort: meta.abort.asSignal(),
+ schema: exchangeScrapeResponseSchema,
+ });
+
+ if (!response.success) {
+ logger.warn("Exchange scrape failed", {
+ ...requestLogContext,
+ scrapeId: meta.id,
+ teamId: meta.internalOptions.teamId,
+ errorCode: response.error?.code,
+ durationMs: Date.now() - startTime,
+ });
+ throw new EngineError("Exchange request failed");
+ }
+
+ const responseLogContext = getExchangeResponseLogContext(
+ response.data.metadata,
+ );
+
+ logger.info("Exchange scrape completed", {
+ ...requestLogContext,
+ ...responseLogContext,
+ scrapeId: meta.id,
+ teamId: meta.internalOptions.teamId,
+ integrationId: response.data.source?.provider,
+ accessEventId: response.accessEventId,
+ creditsCost: response.creditsCost,
+ durationMs: Date.now() - startTime,
+ });
+
+ setSpanAttributes(span, {
+ "exchange.integration_id": response.data.source?.provider,
+ "exchange.credits_cost": response.creditsCost,
+ "exchange.cache_state": responseLogContext.cacheState,
+ "exchange.cache_age_ms": responseLogContext.cacheAgeMs,
+ "exchange.duration_ms": Date.now() - startTime,
+ });
+
+ return {
+ url: response.data.url ?? url,
+ html: buildMetadataHtml(response.data.title, response.data.description),
+ markdown: response.data.markdown,
+ json: response.data.json,
+ statusCode: 200,
+ contentType: "text/markdown",
+ proxyUsed: "basic",
+ exchange: {
+ handled: true,
+ creditsCost: response.creditsCost,
+ ...(response.accessEventId === undefined
+ ? {}
+ : { accessEventId: response.accessEventId }),
+ ...(response.data.source?.provider === undefined
+ ? {}
+ : { integrationId: response.data.source.provider }),
+ },
+ };
+ } catch (error) {
+ logger.warn("Exchange scrape errored", {
+ ...requestLogContext,
+ scrapeId: meta.id,
+ teamId: meta.internalOptions.teamId,
+ durationMs: Date.now() - startTime,
+ errorMessage: error instanceof Error ? error.message : String(error),
+ error,
+ });
+ throw error;
+ }
+ });
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 8a9b2f727..60b986bc7 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -7,6 +7,10 @@ import {
scrapeURLWithFireEngineChromeCDP,
scrapeURLWithFireEngineTLSClient,
} from "./fire-engine";
+import {
+ exchangeMaxReasonableTime,
+ scrapeURLWithExchange,
+} from "./exchange";
import { pdfMaxReasonableTime, scrapePDF } from "./pdf";
import { fetchMaxReasonableTime, scrapeURLWithFetch } from "./fetch";
import {
@@ -30,8 +34,14 @@ import { getPDFMaxPages } from "../../../controllers/v2/types";
import type { PdfMetadata } from "./pdf/types";
import { BrandingProfile } from "../../../types/branding";
import { BrandingNotSupportedError } from "../error";
+import { isUrlBlocked } from "../../WebScraper/utils/blocklist";
+import {
+ canUseExchangeForRequest,
+ type ExchangeScrapeMetadata,
+} from "../../../lib/exchange";
export type Engine =
+ | "exchange"
| "fire-engine;chrome-cdp"
| "fire-engine(retry);chrome-cdp"
| "fire-engine;chrome-cdp;stealth"
@@ -162,11 +172,13 @@ export type EngineScrapeResult = {
proxyUsed: "basic" | "stealth";
timezone?: string;
+ exchange?: ExchangeScrapeMetadata;
};
const engineHandlers: {
[E in Engine]: (meta: Meta) => Promise<EngineScrapeResult>;
} = {
+ "exchange": scrapeURLWithExchange,
index: scrapeURLWithIndex,
"index;documents": scrapeURLWithIndex,
"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,
@@ -186,6 +198,7 @@ const engineHandlers: {
const engineMRTs: {
[E in Engine]: (meta: Meta) => number;
} = {
+ "exchange": exchangeMaxReasonableTime,
index: indexMaxReasonableTime,
"index;documents": indexMaxReasonableTime,
"fire-engine;chrome-cdp": meta =>
@@ -218,6 +231,27 @@ const engineOptions: {
quality: number;
};
} = {
+ "exchange": {
+ features: {
+ actions: false,
+ waitFor: false,
+ screenshot: false,
+ "screenshot@fullScreen": false,
+ pdf: false,
+ document: false,
+ audio: false,
+ video: false,
+ atsv: false,
+ location: false,
+ mobile: false,
+ skipTlsVerification: true,
+ useFastMode: true,
+ stealthProxy: false,
+ branding: false,
+ disableAdblock: false,
+ },
+ quality: 2000,
+ },
index: {
features: {
actions: false,
@@ -547,6 +581,73 @@ export async function buildFallbackList(meta: Meta): Promise<
unsupportedFeatures: Set<FeatureFlag>;
}[]
> {
+ if (
+ !meta.internalOptions.agentIndexOnly &&
+ meta.internalOptions.forceEngine === undefined
+ ) {
+ if (
+ await canUseExchangeForRequest({
+ url: meta.rewrittenUrl ?? meta.url,
+ formats: meta.options.formats,
+ actions: meta.options.actions,
+ headers: meta.options.headers,
+ waitFor: meta.options.waitFor,
+ mobile: meta.options.mobile,
+ location: meta.options.location,
+ proxy: meta.options.proxy,
+ blockAds: meta.options.blockAds,
+ profile: meta.options.profile,
+ atsv: meta.internalOptions.atsv,
+ minAge: meta.options.minAge,
+ includeTags: meta.options.includeTags,
+ excludeTags: meta.options.excludeTags,
+ zeroDataRetention: meta.internalOptions.zeroDataRetention,
+ lockdown: meta.options.lockdown,
+ flags: meta.internalOptions.teamFlags ?? null,
+ })
+ ) {
+ return [
+ {
+ engine: "exchange",
+ unsupportedFeatures: new Set(),
+ },
+ ];
+ }
+
+ // A blocked URL can only have been admitted by the Exchange bypass in
+ // blocklistMiddleware, which only applies to flagged orgs; if the
+ // Exchange is no longer usable by execution time (catalog changed,
+ // service down), fail closed rather than letting normal engines scrape
+ // a blocklisted site. An error here also fails closed: this branch only
+ // runs for flagged orgs, and a retryable scrape failure is preferable
+ // to scraping a potentially blocklisted site with normal engines.
+ if (
+ meta.internalOptions.teamFlags?.professionalProfileCompanyDataBeta ===
+ true
+ ) {
+ try {
+ if (
+ isUrlBlocked(
+ meta.rewrittenUrl ?? meta.url,
+ meta.internalOptions.teamFlags ?? null,
+ {
+ team_id: meta.internalOptions.teamId ?? null,
+ origin: null,
+ },
+ )
+ ) {
+ return [];
+ }
+ } catch (error) {
+ meta.logger.warn(
+ "Exchange blocklist re-check failed; failing closed",
+ { error },
+ );
+ return [];
+ }
+ }
+ }
+
const shouldPrioritizeTlsClient = meta.options.__experimental_engpicker
? (await queryEngpickerVerdict(
meta.options.__experimental_omceDomain ?? new URL(meta.url).hostname,
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 53012521b..3863f972e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -52,6 +52,9 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
!meta.internalOptions.zeroDataRetention &&
meta.winnerEngine !== "index" &&
meta.winnerEngine !== "index;documents" &&
+ // Exchange-delivered content is never stored on the Firecrawl side:
+ // every access must go through the Exchange and its ledger.
+ meta.winnerEngine !== "exchange" &&
!(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers)) &&
!meta.options.parsers?.some(parser => {
if (
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index fb1e738f6..5357e6167 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -87,6 +87,7 @@ import { writeFile } from "node:fs/promises";
import path from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
+import type { ExchangeScrapeMetadata } from "../../lib/exchange";
import {
checkUrl,
type ThreatCheckDedup,
@@ -101,6 +102,7 @@ export type ScrapeUrlResponse =
success: true;
document: Document;
unsupportedFeatures?: Set<FeatureFlag>;
+ exchange?: ExchangeScrapeMetadata;
/**
* Threat protection decisions made for this scrape (initial domain
* check + any redirect re-checks, in order). Read by the billing layer
@@ -1071,6 +1073,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
success: true,
document,
unsupportedFeatures: result.unsupportedFeatures,
+ exchange: engineResult.exchange,
};
});
}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
index 7b350ea42..41a626e68 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
@@ -27,6 +27,11 @@ function shouldIndexForSearch(meta: Meta, document: Document): boolean {
return false;
}
+ // Exchange-delivered content is never stored on the Firecrawl side.
+ if (meta.winnerEngine === "exchange") {
+ return false;
+ }
+
// Lockdown must not forward the target URL to any external service.
if (meta.options.lockdown) {
return false;
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index f3d968e02..41d0b7b20 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -99,6 +99,11 @@ import {
recordMonitorScrapeFailure,
recordMonitorScrapeSuccess,
} from "../monitoring/results";
+import {
+ reportExchangeBilling,
+ warmExchangeCatalog,
+ type ExchangeScrapeMetadata,
+} from "../../lib/exchange";
configDotenv();
@@ -108,6 +113,7 @@ const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;
if (require.main === module) {
cacheableLookup.install(http.globalAgent);
cacheableLookup.install(https.globalAgent);
+ warmExchangeCatalog();
}
async function billScrapeJob(
@@ -118,6 +124,7 @@ async function billScrapeJob(
flags: TeamFlags,
error?: Error | null,
unsupportedFeatures?: Set<FeatureFlag>,
+ exchange?: ExchangeScrapeMetadata,
threatDecisions?: ThreatDecision[],
) {
let creditsToBeBilled: number | null = null;
@@ -146,6 +153,7 @@ async function billScrapeJob(
flags,
error,
unsupportedFeatures,
+ exchange,
threatDecisions,
);
@@ -196,6 +204,18 @@ async function billScrapeJob(
},
);
+ // Reconcile the Exchange ledger. Awaited so the confirmation lands
+ // before any later failure path could attempt to void this access -
+ // the service rejects confirmed->void, making the order decisive.
+ // reportExchangeBilling never throws and is bounded by its timeout.
+ if (exchange?.accessEventId !== undefined) {
+ await reportExchangeBilling({
+ accessEventId: exchange.accessEventId,
+ status: "confirmed",
+ billingReference: billingJobId,
+ });
+ }
+
return creditsToBeBilled;
} catch (error) {
logger.error(
@@ -663,6 +683,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
(await getACUCTeam(job.data.team_id))?.flags ?? null,
undefined,
pipeline.unsupportedFeatures,
+ pipeline.exchange,
pipeline.threatDecisions,
);
@@ -754,6 +775,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
(await getACUCTeam(job.data.team_id))?.flags ?? null,
undefined,
pipeline.unsupportedFeatures,
+ pipeline.exchange,
pipeline.threatDecisions,
);
@@ -954,9 +976,23 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
(await getACUCTeam(job.data.team_id))?.flags ?? null,
error instanceof Error ? error : null,
undefined,
+ undefined,
pipeline?.threatDecisions,
);
+ // The Exchange delivered this access but the scrape ultimately failed,
+ // so the customer was never billed for it - void the access event so
+ // the Exchange ledger reconciles. Fire-and-forget. If billing did
+ // happen before a later step failed, its confirmation was awaited
+ // above and the service rejects confirmed->void, so a paid access can
+ // never be marked void.
+ if (pipeline?.success && pipeline.exchange?.accessEventId !== undefined) {
+ void reportExchangeBilling({
+ accessEventId: pipeline.exchange.accessEventId,
+ status: "void",
+ });
+ }
+
logger.debug("Logging job to DB...");
await logScrape(
{