Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): route eligible scrapes through the Exchange (#4030)
Abimael Martell692d6b7e10fe
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.tsindex 5c2d1cac3..1c4cf0407 100644--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts@@ -6,12 +6,16 @@describe("Audio format engine routing (buildFallbackList)", () => {let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;+ let clearExchangeProvidersForTest: typeof import("../../../lib/exchange.js").clearExchangeProvidersForTest;+ let setExchangeProvidersForTest: typeof import("../../../lib/exchange.js").setExchangeProvidersForTest;const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;const originalIndexUrl = process.env.INDEX_DATABASE_URL;+ const originalExchangeUrl = process.env.FIRE_EXCHANGE_URL;beforeAll(async () => {process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine";+ process.env.FIRE_EXCHANGE_URL = "http://test-exchange";process.env.INDEX_DATABASE_URL ="postgresql://postgres:postgres@localhost:5432/postgres";@@ -20,6 +24,14 @@ describe("Audio format engine routing (buildFallbackList)", () => {({ buildFallbackList } = await import("../../../scraper/scrapeURL/engines/index.js"));+ ({+ clearExchangeProvidersForTest,+ setExchangeProvidersForTest,+ } = await import("../../../lib/exchange.js"));+ });++ afterEach(() => {+ clearExchangeProvidersForTest();});afterAll(() => {@@ -33,6 +45,11 @@ describe("Audio format engine routing (buildFallbackList)", () => {} else {process.env.INDEX_DATABASE_URL = originalIndexUrl;}+ if (originalExchangeUrl === undefined) {+ delete process.env.FIRE_EXCHANGE_URL;+ } else {+ process.env.FIRE_EXCHANGE_URL = originalExchangeUrl;+ }});const buildStubMeta = (featureFlags: string[]) =>@@ -110,4 +127,20 @@ describe("Audio format engine routing (buildFallbackList)", () => {expect(engines).toContain("fire-engine;chrome-cdp");});+ it("does not route agent index-only requests through the Exchange", async () => {+ setExchangeProvidersForTest([+ { id: "acme", routes: [{ domains: ["profiles.example"] }] },+ ]);++ const meta = buildStubMeta([]);+ meta.url = "https://profiles.example/person/example-person";+ meta.options.formats = [{ type: "markdown" }];+ meta.internalOptions.agentIndexOnly = true;+ meta.internalOptions.teamFlags = { professionalProfileCompanyDataBeta: true };++ const fallback = await buildFallbackList(meta);+ const engines = fallback.map(f => f.engine);++ expect(engines).toEqual(["index", "index;documents"]);+ });});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex a372c24e7..9168f7f60 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -165,6 +165,9 @@ const configSchema = z.object({HIGHLIGHT_MODEL_TOKEN: z.string().optional(),HIGHLIGHT_SHADOW_RATE: z.coerce.number().min(0).max(1).default(0),+ // Exchange (routed data sources service)+ FIRE_EXCHANGE_URL: z.url().optional(),+// Fire EngineFIRE_ENGINE_BETA_URL: z.string().optional(),FIRE_ENGINE_STAGING_URL: z.string().optional(),diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 650a4a0f9..a163c46fc 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -34,6 +34,7 @@ import { v2Router } from "./routes/v2";import { nuqShutdown } from "./services/worker/nuq";import { getErrorContactMessage } from "./lib/deployment";import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";+import { warmExchangeCatalog } from "./lib/exchange";import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forcing";import responseTime from "response-time";import { shutdownWebhookQueue } from "./services/webhook";@@ -132,6 +133,7 @@ async function startServer(port = DEFAULT_PORT) {try {await initializeBlocklist();initializeEngineForcing();+ warmExchangeCatalog();} catch (error) {logger.error("Failed to initialize API startup dependencies", {error,diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 935ef612b..aaa59db8e 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -39,6 +39,8 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";// TODO: figure out correct typing for thisconst errorMap: Record<ErrorCodes, any> = {+ // Terms responses are API-level, never transported through workers.+ THIRD_PARTY_DATA_TERMS_REQUIRED: null,SCRAPE_TIMEOUT: ScrapeJobTimeoutError,MAP_TIMEOUT: MapTimeoutError,UNKNOWN_ERROR: UnknownError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 62df2572b..d77a878f8 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -1,4 +1,5 @@export type ErrorCodes =+ | "THIRD_PARTY_DATA_TERMS_REQUIRED"| "SCRAPE_TIMEOUT"| "MAP_TIMEOUT"| "UNKNOWN_ERROR"diff --git a/apps/api/src/lib/exchange-request.ts b/apps/api/src/lib/exchange-request.tsnew file mode 100644index 000000000..17238155b--- /dev/null+++ b/apps/api/src/lib/exchange-request.ts@@ -0,0 +1,47 @@+import {+ getExchangeAccessForRequest,+ type ExchangeAccess,+} from "./exchange";++type ExchangeFlags = Parameters<+ typeof getExchangeAccessForRequest+>[0]["flags"];++/**+ * Run the Exchange access gate against an API request body, sourcing the+ * effective per-page options from a nested scrapeOptions object (crawl+ * bodies) or the body itself (scrape and batch scrape bodies), with v1+ * pageOptions fallbacks.+ */+export function getExchangeAccessForRequestBody(input: {+ body: Record<string, any>;+ flags: ExchangeFlags;+ url: string;+ zeroDataRetention: boolean;+}): Promise<ExchangeAccess> {+ const body = input.body ?? {};+ const scrapeOptions =+ typeof body.scrapeOptions === "object" && body.scrapeOptions !== null+ ? body.scrapeOptions+ : body;++ return getExchangeAccessForRequest({+ url: input.url,+ formats: scrapeOptions.formats,+ actions: scrapeOptions.actions,+ headers: scrapeOptions.headers,+ waitFor: scrapeOptions.waitFor,+ mobile: scrapeOptions.mobile,+ location: scrapeOptions.location,+ proxy: scrapeOptions.proxy,+ blockAds: scrapeOptions.blockAds,+ profile: scrapeOptions.profile,+ atsv: scrapeOptions.atsv ?? body.pageOptions?.atsv,+ minAge: scrapeOptions.minAge,+ includeTags: scrapeOptions.includeTags ?? body.pageOptions?.includeTags,+ excludeTags: scrapeOptions.excludeTags ?? body.pageOptions?.excludeTags,+ zeroDataRetention: input.zeroDataRetention,+ lockdown: scrapeOptions.lockdown ?? body.lockdown,+ flags: input.flags,+ });+}diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.tsnew file mode 100644index 000000000..416e2b627--- /dev/null+++ b/apps/api/src/lib/exchange.test.ts@@ -0,0 +1,589 @@+import { fetch } from "undici";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import { config } from "../config";+import {+ canUseExchangeForRequest,+ clearExchangeProvidersForTest,+ reportExchangeBilling,+ getExchangeAccessForRequest,+ getExchangeRequestLogContext,+ getExchangeResponseLogContext,+ getExchangeSuccessCredits,+ getThirdPartyDataTermsRequiredResponse,+ isSuccessfulExchangeStatusCode,+ isSupportedExchangeFormatRequest,+ resolveExchangeProvider,+ setExchangeProvidersForTest,+} from "./exchange";++vi.mock("undici", () => ({+ fetch: vi.fn(),+}));++const originalConfig = {+ FIRE_EXCHANGE_URL: config.FIRE_EXCHANGE_URL,+ USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,+};++const ACME_TERMS = { key: "acme", version: "2026-01-01" };++const TEST_PROVIDERS = [+ {+ id: "acme",+ creditsCost: 12,+ terms: ACME_TERMS,+ routes: [+ {+ domains: ["profiles.example", "www.profiles.example"],+ pathPrefixes: ["/person/", "/company/"],+ },+ ],+ },+ {+ id: "openfacts",+ creditsCost: 0,+ routes: [{ domains: ["facts.example"] }],+ },+];++const ENABLED_EXCHANGE_FLAGS = {+ professionalProfileCompanyDataBeta: true,+ organizationDataSourceAccess: {+ acme: {+ status: "enabled",+ termsKey: "acme",+ termsVersion: "2026-01-01",+ termsAcceptedAt: "2026-01-01T00:00:00.000Z",+ enabledAt: "2026-01-01T00:00:00.000Z",+ disabledAt: null,+ disabledReason: null,+ },+ },+};++describe("Exchange routing", () => {+ beforeEach(() => {+ vi.mocked(fetch).mockReset();+ config.FIRE_EXCHANGE_URL = "https://exchange.example";+ config.USE_DB_AUTHENTICATION = true;+ setExchangeProvidersForTest(TEST_PROVIDERS);+ });++ afterEach(() => {+ config.FIRE_EXCHANGE_URL = originalConfig.FIRE_EXCHANGE_URL;+ config.USE_DB_AUTHENTICATION = originalConfig.USE_DB_AUTHENTICATION;+ clearExchangeProvidersForTest();+ });++ it("resolves URLs to providers using catalog routes", async () => {+ await expect(+ resolveExchangeProvider(+ "https://profiles.example/person/example-person/details/?trk=foo",+ ),+ ).resolves.toMatchObject({ id: "acme", creditsCost: 12 });+ await expect(+ resolveExchangeProvider("https://www.profiles.example/company/example"),+ ).resolves.toMatchObject({ id: "acme" });+ await expect(+ resolveExchangeProvider("https://facts.example/any/path"),+ ).resolves.toMatchObject({ id: "openfacts" });+ await expect(+ resolveExchangeProvider("https://profiles.example/jobs/example"),+ ).resolves.toBeNull();+ await expect(+ resolveExchangeProvider("https://other.example/person/example"),+ ).resolves.toBeNull();+ await expect(resolveExchangeProvider("not a url")).resolves.toBeNull();++ await expect(+ resolveExchangeProvider("https://profiles.example/person/example-person"),+ ).resolves.not.toBeNull();+ });++ it("respects path segment boundaries for prefixes without trailing slashes", async () => {+ setExchangeProvidersForTest([+ {+ id: "acme",+ creditsCost: 12,+ routes: [{ domains: ["profiles.example"], pathPrefixes: ["/person"] }],+ },+ ]);++ await expect(+ resolveExchangeProvider("https://profiles.example/person"),+ ).resolves.toMatchObject({ id: "acme" });+ await expect(+ resolveExchangeProvider("https://profiles.example/person/example"),+ ).resolves.toMatchObject({ id: "acme" });+ await expect(+ resolveExchangeProvider("https://profiles.example/personality/details"),+ ).resolves.toBeNull();+ await expect(+ resolveExchangeProvider("https://profiles.example/personnel"),+ ).resolves.toBeNull();+ });++ it("serves the stale catalog while refreshing in the background", async () => {+ setExchangeProvidersForTest(TEST_PROVIDERS, -1);+ let resolveFetch: (value: unknown) => void = () => {};+ vi.mocked(fetch).mockReturnValue(+ new Promise(resolve => {+ resolveFetch = resolve;+ }) as ReturnType<typeof fetch>,+ );++ // The catalog is expired and the refresh never resolves during the+ // request - the stale value must be served without waiting.+ await expect(+ resolveExchangeProvider("https://profiles.example/person/example"),+ ).resolves.toMatchObject({ id: "acme" });+ expect(fetch).toHaveBeenCalledTimes(1);++ resolveFetch({ ok: false, status: 503 });+ });++ it("keeps serving the last good catalog when a refresh fails", async () => {+ setExchangeProvidersForTest(TEST_PROVIDERS, -1);+ vi.mocked(fetch).mockResolvedValue({+ ok: false,+ status: 503,+ } as Awaited<ReturnType<typeof fetch>>);++ await expect(+ resolveExchangeProvider("https://profiles.example/person/example"),+ ).resolves.toMatchObject({ id: "acme" });++ // Let the background refresh settle - the failure must not clobber+ // the catalog, and the failure TTL must throttle the next attempt.+ await new Promise(resolve => setTimeout(resolve, 0));+ await expect(+ resolveExchangeProvider("https://profiles.example/person/example"),+ ).resolves.toMatchObject({ id: "acme" });+ expect(fetch).toHaveBeenCalledTimes(1);+ });++ it("degrades to ineligible when the access check itself throws", async () => {+ const explodingFlags = new Proxy(+ {},+ {+ get() {+ throw new Error("flags backend exploded");+ },+ },+ ) as { professionalProfileCompanyDataBeta?: boolean };++ await expect(+ getExchangeAccessForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: explodingFlags,+ }),+ ).resolves.toEqual({ allowed: false, termsRequired: false });+ });++ it("caches failed provider catalog lookups briefly", async () => {+ clearExchangeProvidersForTest();+ vi.mocked(fetch).mockResolvedValue({+ ok: false,+ status: 503,+ } as Awaited<ReturnType<typeof fetch>>);++ await expect(+ resolveExchangeProvider("https://profiles.example/person/example-person"),+ ).resolves.toBeNull();+ await expect(+ resolveExchangeProvider("https://profiles.example/person/example-person"),+ ).resolves.toBeNull();++ expect(fetch).toHaveBeenCalledTimes(1);+ });++ it("parses the provider catalog response", async () => {+ clearExchangeProvidersForTest();+ vi.mocked(fetch).mockResolvedValue({+ ok: true,+ status: 200,+ json: async () => ({+ success: true,+ data: [+ {+ id: "acme",+ name: "Acme",+ description: "Structured records.",+ creditsCost: 12,+ terms: ACME_TERMS,+ capabilities: {+ scrape: {+ resourceTypes: ["records"],+ includes: [],+ urlRoutes: [+ {+ domains: ["Profiles.Example"],+ pathPrefixes: ["person/"],+ },+ ],+ },+ },+ },+ {+ id: "searchonly",+ name: "Search Only",+ description: "No scrape routes.",+ creditsCost: 0,+ capabilities: { search: { modes: ["semantic"] } },+ },+ ],+ }),+ } as unknown as Awaited<ReturnType<typeof fetch>>);++ await expect(+ resolveExchangeProvider("https://profiles.example/person/example"),+ ).resolves.toMatchObject({ id: "acme", terms: ACME_TERMS });+ await expect(+ resolveExchangeProvider("https://searchonly.example/anything"),+ ).resolves.toBeNull();+ expect(fetch).toHaveBeenCalledTimes(1);+ });++ it("builds a compact request log context", () => {+ expect(+ getExchangeRequestLogContext(+ "https://profiles.example/person/example-person/details/?trk=foo",+ ),+ ).toEqual({+ url: "https://profiles.example/person/example-person/details/?trk=foo",+ host: "profiles.example",+ pathPrefix: "person",+ });++ expect(getExchangeRequestLogContext("not a url")).toBeUndefined();++ // Embedded credentials must never reach the logs.+ expect(+ getExchangeRequestLogContext(+ "https://user:[email protected]/person/example",+ ),+ ).toEqual({+ url: "https://profiles.example/person/example",+ host: "profiles.example",+ pathPrefix: "person",+ });+ });++ it("extracts response cache metadata for logs", () => {+ expect(+ getExchangeResponseLogContext({+ cacheState: "hit",+ cachedAt: "2026-06-21T10:00:00.000Z",+ cacheAgeMs: 1000,+ request_id: "req_123",+ extra: "ignored",+ }),+ ).toEqual({+ cacheState: "hit",+ cachedAt: "2026-06-21T10:00:00.000Z",+ cacheAgeMs: 1000,+ providerRequestId: "req_123",+ });++ expect(getExchangeResponseLogContext(null)).toEqual({});+ });++ it("accepts only formats the Exchange can return directly", () => {+ expect(isSupportedExchangeFormatRequest(undefined)).toBe(true);+ expect(isSupportedExchangeFormatRequest([{ type: "markdown" }])).toBe(+ true,+ );+ expect(isSupportedExchangeFormatRequest(["json"])).toBe(true);+ expect(+ isSupportedExchangeFormatRequest([+ { type: "markdown" },+ { type: "json" },+ ]),+ ).toBe(true);+ expect(isSupportedExchangeFormatRequest([{ type: "html" }])).toBe(false);+ // deterministicJson extractors run against page HTML, which Exchange+ // responses do not carry.+ expect(isSupportedExchangeFormatRequest([{ type: "deterministicJson" }])).toBe(+ false,+ );+ expect(isSupportedExchangeFormatRequest([])).toBe(false);+ });++ it("allows eligible requests when access and terms are current", async () => {+ await expect(+ getExchangeAccessForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: ENABLED_EXCHANGE_FLAGS,+ }),+ ).resolves.toMatchObject({+ allowed: true,+ provider: { id: "acme", creditsCost: 12 },+ });++ await expect(+ canUseExchangeForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "json" }],+ actions: [{ type: "wait" }],+ flags: ENABLED_EXCHANGE_FLAGS,+ }),+ ).resolves.toBe(false);++ await expect(+ canUseExchangeForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "json" }],+ zeroDataRetention: true,+ flags: ENABLED_EXCHANGE_FLAGS,+ }),+ ).resolves.toBe(false);++ // Profile-backed scrapes expect session-specific content.+ await expect(+ canUseExchangeForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ profile: { id: "profile-1" },+ flags: ENABLED_EXCHANGE_FLAGS,+ }),+ ).resolves.toBe(false);++ // atsv requests stay on engines that support the flag.+ await expect(+ canUseExchangeForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ atsv: true,+ flags: ENABLED_EXCHANGE_FLAGS,+ }),+ ).resolves.toBe(false);++ // minAge asks for Firecrawl-cached data, which the Exchange never has.+ await expect(+ canUseExchangeForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ minAge: 3_600_000,+ flags: ENABLED_EXCHANGE_FLAGS,+ }),+ ).resolves.toBe(false);++ // Selector-based filtering does not apply to provider records.+ await expect(+ canUseExchangeForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ includeTags: ["article"],+ flags: ENABLED_EXCHANGE_FLAGS,+ }),+ ).resolves.toBe(false);+ await expect(+ canUseExchangeForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ excludeTags: ["nav"],+ flags: ENABLED_EXCHANGE_FLAGS,+ }),+ ).resolves.toBe(false);+ });++ it("requires the provider's terms before routing", async () => {+ const access = await getExchangeAccessForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: { professionalProfileCompanyDataBeta: true },+ });++ expect(access).toEqual({+ allowed: false,+ termsRequired: true,+ terms: ACME_TERMS,+ });++ expect(getThirdPartyDataTermsRequiredResponse(ACME_TERMS)).toMatchObject({+ success: false,+ code: "THIRD_PARTY_DATA_TERMS_REQUIRED",+ requiresAction: {+ type: "accept_terms",+ terms: "acme",+ version: "2026-01-01",+ },+ });+ });++ it("requires current terms when the accepted version is stale", async () => {+ await expect(+ getExchangeAccessForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: {+ professionalProfileCompanyDataBeta: true,+ organizationDataSourceAccess: {+ acme: {+ status: "enabled",+ termsKey: "acme",+ termsVersion: "2025-12-01",+ },+ },+ },+ }),+ ).resolves.toEqual({+ allowed: false,+ termsRequired: true,+ terms: ACME_TERMS,+ });+ });++ it("does not route or prompt for terms when access is disabled", async () => {+ await expect(+ getExchangeAccessForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: {+ professionalProfileCompanyDataBeta: true,+ organizationDataSourceAccess: {+ acme: {+ status: "disabled",+ termsKey: "acme",+ termsVersion: "2026-01-01",+ disabledAt: "2026-01-02T00:00:00.000Z",+ disabledReason: "customer_disabled",+ },+ },+ },+ }),+ ).resolves.toEqual({ allowed: false, termsRequired: false });+ });++ it("allows providers with no declared terms without acceptance", async () => {+ await expect(+ getExchangeAccessForRequest({+ url: "https://facts.example/records/1",+ formats: [{ type: "markdown" }],+ flags: { professionalProfileCompanyDataBeta: true },+ }),+ ).resolves.toMatchObject({ allowed: true, provider: { id: "openfacts" } });++ await expect(+ getExchangeAccessForRequest({+ url: "https://facts.example/records/1",+ formats: [{ type: "markdown" }],+ flags: {+ professionalProfileCompanyDataBeta: true,+ organizationDataSourceAccess: {+ openfacts: { status: "disabled" },+ },+ },+ }),+ ).resolves.toEqual({ allowed: false, termsRequired: false });+ });++ it("does not route unless the beta flag is enabled", async () => {+ await expect(+ canUseExchangeForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ }),+ ).resolves.toBe(false);++ await expect(+ canUseExchangeForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: { professionalProfileCompanyDataBeta: false },+ }),+ ).resolves.toBe(false);+ });++ it("does not route unless the Exchange is configured", async () => {+ config.FIRE_EXCHANGE_URL = undefined;++ await expect(+ canUseExchangeForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: ENABLED_EXCHANGE_FLAGS,+ }),+ ).resolves.toBe(false);+ });++ it("bills the reported credit cost only for successful handled responses", () => {+ expect(isSuccessfulExchangeStatusCode(200)).toBe(true);+ expect(isSuccessfulExchangeStatusCode(204)).toBe(true);+ expect(isSuccessfulExchangeStatusCode(304)).toBe(true);+ expect(isSuccessfulExchangeStatusCode(404)).toBe(false);++ expect(+ getExchangeSuccessCredits({+ exchange: { handled: true, creditsCost: 12 },+ statusCode: 200,+ }),+ ).toBe(12);++ expect(+ getExchangeSuccessCredits({+ exchange: { handled: true, creditsCost: 0 },+ statusCode: 304,+ }),+ ).toBe(0);++ expect(+ getExchangeSuccessCredits({+ exchange: { handled: true, creditsCost: 12 },+ statusCode: 404,+ }),+ ).toBeNull();++ expect(+ getExchangeSuccessCredits({+ statusCode: 200,+ }),+ ).toBeNull();+ });++ it("reports billing outcomes without throwing on service failures", async () => {+ vi.mocked(fetch).mockResolvedValue({+ ok: true,+ status: 200,+ } as Awaited<ReturnType<typeof fetch>>);++ await reportExchangeBilling({+ accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",+ status: "confirmed",+ billingReference: "bill-1",+ });++ expect(fetch).toHaveBeenCalledWith(+ "https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing",+ expect.objectContaining({+ method: "POST",+ body: JSON.stringify({ status: "confirmed", billingReference: "bill-1" }),+ }),+ );++ await reportExchangeBilling({+ accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",+ status: "void",+ });++ expect(fetch).toHaveBeenLastCalledWith(+ "https://exchange.example/v1/access-events/6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60/billing",+ expect.objectContaining({+ method: "POST",+ body: JSON.stringify({ status: "void" }),+ }),+ );++ vi.mocked(fetch).mockRejectedValue(new Error("connect timeout"));+ await expect(+ reportExchangeBilling({+ accessEventId: "6f1f5aab-3f78-4d0a-8a3d-2b1d3c4e5f60",+ status: "confirmed",+ billingReference: "bill-1",+ }),+ ).resolves.toBeUndefined();+ });+});diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.tsnew file mode 100644index 000000000..9de05a473--- /dev/null+++ b/apps/api/src/lib/exchange.ts@@ -0,0 +1,639 @@+import { fetch } from "undici";+import { z } from "zod";++import { config } from "../config";+import type { FormatObject } from "../controllers/v2/types";+import { logger as rootLogger } from "./logger";++type OrganizationDataSourceAccessRecord = {+ status?: string | null;+ termsKey?: string | null;+ termsVersion?: string | null;+ termsAcceptedAt?: string | null;+ enabledAt?: string | null;+ disabledAt?: string | null;+ disabledReason?: string | null;+};++type OrganizationDataSourceAccess = Record<+ string,+ OrganizationDataSourceAccessRecord | null | undefined+>;++type RouteInput = {+ url: string;+ formats?: FormatObject[] | unknown[];+ actions?: unknown[];+ headers?: Record<string, unknown>;+ waitFor?: number;+ mobile?: boolean;+ location?: unknown;+ proxy?: unknown;+ blockAds?: boolean;+ profile?: unknown;+ atsv?: boolean;+ minAge?: number;+ includeTags?: unknown[];+ excludeTags?: unknown[];+ zeroDataRetention?: boolean;+ lockdown?: boolean;+ flags?: {+ professionalProfileCompanyDataBeta?: boolean;+ organizationDataSourceAccess?: OrganizationDataSourceAccess | null;+ } | null;+};++export type ExchangeScrapeMetadata = {+ handled: true;+ creditsCost: number;+ accessEventId?: string;+ integrationId?: string;+};++export type ExchangeTerms = {+ key: string;+ version: string;+};++export type ExchangeProvider = {+ id: string;+ creditsCost: number;+ terms?: ExchangeTerms;+ routes: {+ domains: Set<string>;+ pathPrefixes: string[];+ }[];+};++// deterministicJson is deliberately unsupported: its extractor scripts run+// against page HTML, which Exchange responses do not carry.+const SUPPORTED_FORMATS = new Set(["markdown", "json"]);+const EXCHANGE_BETA_FLAG = "professionalProfileCompanyDataBeta";+const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED";+const THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE =+ "An organization admin must accept this data source's terms before this URL can be processed.";++const EXCHANGE_PROVIDERS_PATH = "/v1/providers";+const EXCHANGE_PROVIDERS_TIMEOUT_MS = 2_000;+const EXCHANGE_PROVIDERS_TTL_MS = 60_000;+const EXCHANGE_PROVIDERS_FAILURE_TTL_MS = 30_000;++const exchangeProvidersSchema = z.object({+ success: z.literal(true),+ data: z.array(+ z+ .object({+ id: z.string(),+ // No .catch() here: a malformed credit cost must reject the catalog+ // (keeping the last good one) rather than silently billing 0.+ creditsCost: z.number().int().nonnegative(),+ terms: z+ .object({+ key: z.string(),+ version: z.string(),+ })+ .optional(),+ capabilities: z+ .object({+ scrape: z+ .object({+ urlRoutes: z+ .array(+ z+ .object({+ domains: z.string().array(),+ pathPrefixes: z.string().array(),+ })+ .passthrough(),+ )+ .optional(),+ })+ .passthrough()+ .optional(),+ })+ .passthrough(),+ })+ .passthrough(),+ ),+});++let cachedProviders:+ | {+ expiresAt: number;+ value: ExchangeProvider[] | null;+ }+ | undefined;+let providersRequest: Promise<ExchangeProvider[] | null> | undefined;++function normalizeHost(host: string): string {+ return host.trim().toLowerCase().replace(/\.$/, "");+}++function normalizePathPrefix(prefix: string): string {+ const trimmed = prefix.trim();+ if (!trimmed) {+ return "/";+ }++ return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;+}++function getExchangeBaseUrl(): string | null {+ if (!config.FIRE_EXCHANGE_URL) {+ return null;+ }++ return config.FIRE_EXCHANGE_URL.replace(/\/+$/, "");+}++function normalizeProviders(+ raw: z.infer<typeof exchangeProvidersSchema>,+): ExchangeProvider[] {+ return raw.data+ .map(provider => ({+ id: provider.id,+ creditsCost: provider.creditsCost,+ ...(provider.terms === undefined ? {} : { terms: provider.terms }),+ routes: (provider.capabilities.scrape?.urlRoutes ?? []).map(route => ({+ domains: new Set(route.domains.map(normalizeHost)),+ pathPrefixes: route.pathPrefixes.map(normalizePathPrefix),+ })),+ }))+ .filter(provider => provider.routes.length > 0);+}++async function fetchExchangeProviders(): Promise<ExchangeProvider[] | null> {+ const baseUrl = getExchangeBaseUrl();+ if (!baseUrl) {+ return null;+ }++ try {+ const response = await fetch(`${baseUrl}${EXCHANGE_PROVIDERS_PATH}`, {+ method: "GET",+ signal: AbortSignal.timeout(EXCHANGE_PROVIDERS_TIMEOUT_MS),+ });++ if (!response.ok) {+ rootLogger.warn("Exchange providers request failed", {+ statusCode: response.status,+ });+ return null;+ }++ const parsed = exchangeProvidersSchema.parse(await response.json());+ return normalizeProviders(parsed);+ } catch (error) {+ rootLogger.warn("Exchange providers request errored", { error });+ return null;+ }+}++async function getExchangeProviders(): Promise<ExchangeProvider[] | null> {+ if (cachedProviders && cachedProviders.expiresAt > Date.now()) {+ return cachedProviders.value;+ }++ if (!providersRequest) {+ providersRequest = fetchExchangeProviders()+ .then(providers => {+ if (providers === null) {+ // Keep serving the last good catalog through transient outages;+ // the failure TTL only delays the next refresh attempt.+ cachedProviders = {+ value: cachedProviders?.value ?? null,+ expiresAt: Date.now() + EXCHANGE_PROVIDERS_FAILURE_TTL_MS,+ };+ } else {+ cachedProviders = {+ value: providers,+ expiresAt: Date.now() + EXCHANGE_PROVIDERS_TTL_MS,+ };+ }+ return cachedProviders.value;+ })+ .finally(() => {+ providersRequest = undefined;+ });+ }++ // Serve the stale catalog while the refresh runs in the background so+ // request latency never depends on the catalog endpoint; only the very+ // first lookup after boot has nothing to serve and waits.+ if (cachedProviders) {+ return cachedProviders.value;+ }++ return providersRequest;+}++function providerMatchesUrl(provider: ExchangeProvider, inputUrl: string): boolean {+ let parsed: URL;+ try {+ parsed = new URL(inputUrl);+ } catch {+ return false;+ }++ const host = normalizeHost(parsed.hostname);+ const pathname = parsed.pathname || "/";++ return provider.routes.some(route => {+ if (!route.domains.has(host)) {+ return false;+ }++ if (route.pathPrefixes.length === 0) {+ return true;+ }++ // Prefix matches respect path segment boundaries: "/person" matches+ // "/person" and "/person/x" but never "/personality".+ return route.pathPrefixes.some(prefix =>+ prefix.endsWith("/")+ ? pathname.startsWith(prefix)+ : pathname === prefix || pathname.startsWith(`${prefix}/`),+ );+ });+}++export async function resolveExchangeProvider(+ inputUrl: string,+): Promise<ExchangeProvider | null> {+ const providers = await getExchangeProviders();+ if (providers === null) {+ return null;+ }++ return providers.find(provider => providerMatchesUrl(provider, inputUrl)) ?? null;+}++export function getExchangeRequestLogContext(inputUrl: string):+ | {+ url: string;+ host: string;+ pathPrefix: string | null;+ }+ | undefined {+ let parsed: URL;+ try {+ parsed = new URL(inputUrl);+ } catch {+ return undefined;+ }++ // Never log embedded credentials from user-submitted URLs.+ parsed.username = "";+ parsed.password = "";++ return {+ url: parsed.href,+ host: parsed.hostname.toLowerCase(),+ pathPrefix:+ parsed.pathname+ .split("/")+ .map(part => part.trim())+ .filter(part => part.length > 0)[0] ?? null,+ };+}++export function getExchangeResponseLogContext(meta: unknown): {+ cacheState?: string;+ cachedAt?: string;+ cacheAgeMs?: number;+ providerRequestId?: string;+} {+ if (typeof meta !== "object" || meta === null) {+ return {};+ }++ const record = meta as Record<string, unknown>;+ const requestId = record.request_id ?? record.requestId;++ return {+ ...(typeof record.cacheState === "string"+ ? { cacheState: record.cacheState }+ : {}),+ ...(typeof record.cachedAt === "string"+ ? { cachedAt: record.cachedAt }+ : {}),+ ...(typeof record.cacheAgeMs === "number"+ ? { cacheAgeMs: record.cacheAgeMs }+ : {}),+ ...(typeof requestId === "string" ? { providerRequestId: requestId } : {}),+ };+}++export function isSuccessfulExchangeStatusCode(statusCode: number): boolean {+ return (statusCode >= 200 && statusCode < 300) || statusCode === 304;+}++export function isSupportedExchangeFormatRequest(+ formats?: FormatObject[] | unknown[],+): boolean {+ if (formats === undefined) {+ return true;+ }++ if (!Array.isArray(formats) || formats.length === 0) {+ return false;+ }++ return formats.every(format => {+ const type =+ typeof format === "string"+ ? format+ : typeof format === "object" && format !== null && "type" in format+ ? (format as { type?: unknown }).type+ : undefined;++ return typeof type === "string" && SUPPORTED_FORMATS.has(type);+ });+}++type DataSourceAccessDecision = "allowed" | "terms_required" | "not_enabled";++function getProviderAccessDecision(+ provider: ExchangeProvider,+ flags: RouteInput["flags"],+): DataSourceAccessDecision {+ if (config.USE_DB_AUTHENTICATION !== true) {+ return "allowed";+ }++ const access = flags?.organizationDataSourceAccess?.[provider.id];+ const entry = typeof access === "object" && access !== null ? access : null;++ if (provider.terms === undefined) {+ return entry !== null && entry.status !== "enabled" ? "not_enabled" : "allowed";+ }++ if (entry === null) {+ return "terms_required";+ }++ if (entry.status !== "enabled") {+ return "not_enabled";+ }++ return entry.termsKey === provider.terms.key &&+ entry.termsVersion === provider.terms.version+ ? "allowed"+ : "terms_required";+}++function isExchangeEligibleRequest(input: RouteInput): boolean {+ if (input.flags?.[EXCHANGE_BETA_FLAG] !== true) {+ return false;+ }++ if (!config.FIRE_EXCHANGE_URL) {+ return false;+ }++ if (!input.url) {+ return false;+ }++ if (input.zeroDataRetention || input.lockdown) {+ return false;+ }++ if (Array.isArray(input.actions) && input.actions.length > 0) {+ return false;+ }++ if (input.headers && Object.keys(input.headers).length > 0) {+ return false;+ }++ if (input.waitFor !== undefined && input.waitFor !== 0) {+ return false;+ }++ if (input.mobile || input.location || input.blockAds === false) {+ return false;+ }++ // Profile-backed scrapes expect session-specific content, which the+ // Exchange cannot serve.+ if (input.profile !== undefined) {+ return false;+ }++ // atsv is only supported by browser engines; requests that set it keep an+ // engine that can honor it instead of routing to the Exchange.+ if (input.atsv === true) {+ return false;+ }++ // minAge requests ask for Firecrawl-cached data; the Exchange serves+ // provider data and Firecrawl never caches it, so the semantics cannot+ // be honored here.+ if (input.minAge !== undefined) {+ return false;+ }++ // Selector-based content filtering does not apply to provider records.+ if (+ (Array.isArray(input.includeTags) && input.includeTags.length > 0) ||+ (Array.isArray(input.excludeTags) && input.excludeTags.length > 0)+ ) {+ return false;+ }++ if (input.proxy === "stealth" || input.proxy === "enhanced") {+ return false;+ }++ if (!isSupportedExchangeFormatRequest(input.formats)) {+ return false;+ }++ return true;+}++export type ExchangeAccess =+ | {+ allowed: true;+ termsRequired: false;+ provider: ExchangeProvider;+ }+ | {+ allowed: false;+ termsRequired: true;+ terms: ExchangeTerms;+ }+ | {+ allowed: false;+ termsRequired: false;+ };++export async function getExchangeAccessForRequest(+ input: RouteInput,+): Promise<ExchangeAccess> {+ // The Exchange gate sits on the hot path of every scrape request; no+ // failure inside it may ever fail the request itself. Anything unexpected+ // degrades to "not eligible" and the request continues on the normal path.+ try {+ if (!isExchangeEligibleRequest(input)) {+ return { allowed: false, termsRequired: false };+ }++ const provider = await resolveExchangeProvider(input.url);+ if (provider === null) {+ return { allowed: false, termsRequired: false };+ }++ const decision = getProviderAccessDecision(provider, input.flags);+ if (decision === "terms_required" && provider.terms !== undefined) {+ return { allowed: false, termsRequired: true, terms: provider.terms };+ }+ if (decision !== "allowed") {+ return { allowed: false, termsRequired: false };+ }++ return { allowed: true, termsRequired: false, provider };+ } catch (error) {+ rootLogger.warn("Exchange access check errored; treating as ineligible", {+ error,+ });+ return { allowed: false, termsRequired: false };+ }+}++export async function canUseExchangeForRequest(+ input: RouteInput,+): Promise<boolean> {+ return (await getExchangeAccessForRequest(input)).allowed;+}++function getThirdPartyDataTermsSettingsUrl(): string {+ return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/settings?tab=data-sources`;+}++export function getThirdPartyDataTermsRequiredResponse(terms: ExchangeTerms) {+ return {+ success: false as const,+ code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE as "THIRD_PARTY_DATA_TERMS_REQUIRED",+ error: THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE,+ requiresAction: {+ type: "accept_terms",+ terms: terms.key,+ version: terms.version,+ url: getThirdPartyDataTermsSettingsUrl(),+ },+ };+}++export function getExchangeSuccessCredits(input: {+ exchange?: ExchangeScrapeMetadata;+ statusCode?: number | null;+}): number | null {+ if (input.exchange?.handled !== true) {+ return null;+ }++ const statusCode = input.statusCode;+ if (+ statusCode === undefined ||+ statusCode === null ||+ !isSuccessfulExchangeStatusCode(statusCode)+ ) {+ return null;+ }++ return input.exchange.creditsCost;+}++const EXCHANGE_BILLING_TIMEOUT_MS = 5_000;++/**+ * Report the billing outcome of a delivered Exchange access so the service+ * can reconcile its ledger: "confirmed" once the customer was billed, "void"+ * when the delivered access was ultimately discarded and never billed.+ * Failures only log - the service flags unresolved events for follow-up.+ */+export async function reportExchangeBilling(input: {+ accessEventId: string;+ status: "confirmed" | "void";+ billingReference?: string;+}): Promise<void> {+ const baseUrl = getExchangeBaseUrl();+ if (!baseUrl) {+ return;+ }++ try {+ const response = await fetch(+ `${baseUrl}/v1/access-events/${encodeURIComponent(input.accessEventId)}/billing`,+ {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({+ status: input.status,+ ...(input.billingReference === undefined+ ? {}+ : { billingReference: input.billingReference }),+ }),+ signal: AbortSignal.timeout(EXCHANGE_BILLING_TIMEOUT_MS),+ },+ );++ if (!response.ok) {+ rootLogger.warn("Exchange billing report failed", {+ accessEventId: input.accessEventId,+ status: input.status,+ statusCode: response.status,+ });+ }+ } catch (error) {+ rootLogger.warn("Exchange billing report errored", {+ accessEventId: input.accessEventId,+ status: input.status,+ error,+ });+ }+}++/**+ * Warm the provider catalog at process startup so the first flagged-org+ * request never waits on the fetch; after this, stale-while-revalidate+ * keeps every lookup in-memory. No-op when the Exchange is not configured;+ * never throws.+ */+export function warmExchangeCatalog(): void {+ if (!config.FIRE_EXCHANGE_URL) {+ return;+ }++ void getExchangeProviders();+}++export function setExchangeProvidersForTest(+ providers: {+ id: string;+ creditsCost?: number;+ terms?: ExchangeTerms;+ routes: { domains: string[]; pathPrefixes?: string[] }[];+ }[],+ ttlMs = 300_000,+) {+ cachedProviders = {+ value: providers.map(provider => ({+ id: provider.id,+ creditsCost: provider.creditsCost ?? 0,+ ...(provider.terms === undefined ? {} : { terms: provider.terms }),+ routes: provider.routes.map(route => ({+ domains: new Set(route.domains.map(normalizeHost)),+ pathPrefixes: (route.pathPrefixes ?? []).map(normalizePathPrefix),+ })),+ })),+ expiresAt: Date.now() + ttlMs,+ };+}++export function clearExchangeProvidersForTest() {+ cachedProviders = undefined;+ providersRequest = undefined;+}diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex a6c67caee..9c1f9e768 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -6,6 +6,33 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";import type { ThreatDecision } from "./threat-protection/types";describe("calculateCreditsToBeBilled", () => {+ it("bills handled Exchange successes at the reported credit cost", async () => {+ const credits = await calculateCreditsToBeBilled(+ {+ formats: [{ type: "markdown" }],+ } as any,+ {+ teamId: "team-id",+ },+ {+ metadata: {+ statusCode: 200,+ url: "https://profiles.example/person/example-person",+ proxyUsed: "basic",+ },+ } as any,+ {+ totalCost: 0,+ } as any,+ {} as any,+ undefined,+ undefined,+ { handled: true, creditsCost: 12 },+ );++ expect(credits).toBe(12);+ });+it("bills X/Twitter scrapes at 30 credits", async () => {const credits = await calculateCreditsToBeBilled({@@ -142,6 +169,7 @@ const billWithDecisions = (args: {{} as any,args.error,undefined,+ undefined,args.threatDecisions,);diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex b393646d7..58347aec3 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -10,6 +10,10 @@ import { hasFormatOfType } from "./format-utils";import { TransportableError } from "./error";import { FeatureFlag } from "../scraper/scrapeURL/engines";import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";+import {+ ExchangeScrapeMetadata,+ getExchangeSuccessCredits,+} from "./exchange";import type { ThreatDecision } from "./threat-protection/types";import { UnsafeDomainBlockedError } from "./threat-protection/error";@@ -64,6 +68,7 @@ export async function calculateCreditsToBeBilled(flags: TeamFlags,error?: Error | null,unsupportedFeatures?: Set<FeatureFlag>,+ exchange?: ExchangeScrapeMetadata,// Threat protection decisions for this scrape (initial + redirect checks,// in order). Each decision with `providerConsulted` bills a scan fee (+2// per unique scanned URL) on top of the scrape's own cost — on both success@@ -112,6 +117,14 @@ export async function calculateCreditsToBeBilled(return creditsToBeBilled + threatScanCredits;}+ const exchangeCredits = getExchangeSuccessCredits({+ exchange,+ statusCode: document.metadata?.statusCode,+ });+ if (exchangeCredits !== null) {+ return exchangeCredits + threatScanCredits;+ }+let creditsToBeBilled = 1; // Assuming 1 credit per documentif (options.lockdown) {diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 5528a9aa8..73f553e3b 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -29,6 +29,9 @@ import {CREDITS_FEATURE_ID,} from "../services/autumn/autumn.service";import { getTeamBalance } from "../services/autumn/usage";+import { getThirdPartyDataTermsRequiredResponse } from "../lib/exchange";+import { getExchangeAccessForRequestBody } from "../lib/exchange-request";+import { getScrapeZDR } from "../lib/zdr-helpers";export function checkCreditsMiddleware(_minimum?: number,@@ -284,10 +287,58 @@ export function blocklistMiddleware(req: RequestWithMaybeACUC<any, any, any>,res: Response,next: NextFunction,+) {+ return blocklistGate(req, res, next, { exchange: false });+}++/**+ * Blocklist gate for single-URL scrape-shaped routes (scrape, crawl), where+ * an Exchange-eligible URL may bypass the blocklist because the exchange+ * engine can serve it. Everything else (map, search, batch scrape, monitors)+ * keeps plain blocklist behavior - batch stays out until its jobs carry the+ * access flags the worker-side recheck needs.+ */+export function scrapeBlocklistMiddleware(+ req: RequestWithMaybeACUC<any, any, any>,+ res: Response,+ next: NextFunction,+) {+ return blocklistGate(req, res, next, { exchange: true });+}++function blocklistGate(+ req: RequestWithMaybeACUC<any, any, any>,+ res: Response,+ next: NextFunction,+ options: { exchange: boolean },) {(async () => {+ const zeroDataRetention =+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ req.body?.zeroDataRetention === true;+ const exchangeAccess =+ options.exchange &&+ typeof req.body.url === "string" &&+ (await getExchangeAccessForRequestBody({+ body: req.body,+ flags: req.acuc?.flags ?? null,+ url: req.body.url,+ zeroDataRetention,+ }));+ const canUseExchange =+ typeof exchangeAccess === "object" && exchangeAccess.allowed;++ if (typeof exchangeAccess === "object" && exchangeAccess.termsRequired) {+ if (!res.headersSent) {+ return res+ .status(403)+ .json(getThirdPartyDataTermsRequiredResponse(exchangeAccess.terms));+ }+ }+if (typeof req.body.url === "string" &&+ !canUseExchange &&isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {team_id: req.acuc?.team_id ?? null,origin: typeof req.body.origin === "string" ? req.body.origin : null,diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex bc92811d1..d2cdf6dff 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -28,6 +28,7 @@ import {authMiddleware,checkCreditsMiddleware,blocklistMiddleware,+ scrapeBlocklistMiddleware,countryCheck,idempotencyMiddleware,requestTimingMiddleware,@@ -51,7 +52,7 @@ v1Router.post(authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),countryCheck,checkCreditsMiddleware(1),- blocklistMiddleware,+ scrapeBlocklistMiddleware,wrap(scrapeController),);@@ -60,7 +61,7 @@ v1Router.post(authMiddleware(RateLimiterMode.Crawl),countryCheck,checkCreditsMiddleware(),- blocklistMiddleware,+ scrapeBlocklistMiddleware,idempotencyMiddleware,wrap(crawlController),);diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex e3a0991dd..31c9b3997 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -38,6 +38,7 @@ import {authMiddleware,checkCreditsMiddleware,blocklistMiddleware,+ scrapeBlocklistMiddleware,countryCheck,idempotencyMiddleware,requestTimingMiddleware,@@ -210,7 +211,7 @@ v2Router.post(authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),countryCheck,checkCreditsMiddleware(1),- blocklistMiddleware,+ scrapeBlocklistMiddleware,wrap(scrapeController),);@@ -257,7 +258,7 @@ v2Router.post(authMiddleware(RateLimiterMode.Crawl),countryCheck,checkCreditsMiddleware(),- blocklistMiddleware,+ scrapeBlocklistMiddleware,idempotencyMiddleware,wrap(crawlController),);diff --git a/apps/api/src/scraper/scrapeURL/engines/exchange.ts b/apps/api/src/scraper/scrapeURL/engines/exchange.tsnew file mode 100644index 000000000..c6c41de2d--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/exchange.ts@@ -0,0 +1,186 @@+import { z } from "zod";++import { Meta } from "..";+import { EngineScrapeResult } from ".";+import { config } from "../../../config";+import {+ getExchangeRequestLogContext,+ getExchangeResponseLogContext,+} from "../../../lib/exchange";+import { setSpanAttributes, withSpan } from "../../../lib/otel-tracer";+import { robustFetch } from "../lib/fetch";+import { EngineError } from "../error";++const exchangeScrapeResponseSchema = z.union([+ z+ .object({+ success: z.literal(true),+ accessEventId: z.string().optional(),+ // No .catch() here: a malformed credit cost must fail the scrape+ // loudly rather than silently billing 0 for a delivered access.+ creditsCost: z.number().int().nonnegative(),+ data: z+ .object({+ url: z.string().optional(),+ title: z.string().optional(),+ description: z.string().optional(),+ source: z+ .object({+ provider: z.string().optional(),+ })+ .passthrough()+ .optional(),+ metadata: z.record(z.string(), z.unknown()).optional(),+ markdown: z.string().optional(),+ json: z.unknown().optional(),+ })+ .passthrough(),+ })+ .passthrough(),+ z+ .object({+ success: z.literal(false),+ error: z+ .object({+ code: z.string().optional(),+ message: z.string().optional(),+ })+ .passthrough()+ .optional(),+ })+ .passthrough(),+]);++export function exchangeMaxReasonableTime(meta: Meta): number {+ return meta.options.timeout ?? 60_000;+}++function escapeHtml(value: string): string {+ return value+ .replace(/&/g, "&")+ .replace(/</g, "<")+ .replace(/>/g, ">")+ .replace(/"/g, """);+}++// Exchange responses carry no page HTML; synthesize a minimal head so the+// metadata transformer can populate the document's title and description.+function buildMetadataHtml(title?: string, description?: string): string {+ const titleTag = title === undefined ? "" : `<title>${escapeHtml(title)}</title>`;+ const descriptionTag =+ description === undefined+ ? ""+ : `<meta name="description" content="${escapeHtml(description)}">`;+ return `<!DOCTYPE html><html><head>${titleTag}${descriptionTag}</head><body></body></html>`;+}++export async function scrapeURLWithExchange(+ meta: Meta,+): Promise<EngineScrapeResult> {+ return withSpan("engine.exchange.scrape", async span => {+ const startTime = Date.now();+ const url = meta.rewrittenUrl ?? meta.url;+ const requestLogContext = getExchangeRequestLogContext(url);+ const logger = meta.logger.child({ method: "scrapeURLWithExchange" });++ setSpanAttributes(span, {+ "engine.type": "exchange",+ // Follow the same credential redaction as the log context.+ "engine.url": requestLogContext?.url ?? "",+ "engine.team_id": meta.internalOptions.teamId,+ });++ logger.info("Exchange scrape started", {+ ...requestLogContext,+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId,+ maxAge: meta.options.maxAge,+ });++ try {+ const response = await robustFetch({+ url: `${config.FIRE_EXCHANGE_URL!.replace(/\/+$/, "")}/v1/scrape`,+ method: "POST",+ body: {+ requestId: meta.id,+ teamId: meta.internalOptions.teamId,+ url,+ formats: ["markdown", "json"],+ ...(meta.options.maxAge === undefined+ ? {}+ : { maxAge: meta.options.maxAge }),+ },+ logger: logger.child({ method: "exchangeScrape/robustFetch" }),+ tryCount: 2,+ ignoreFailureStatus: true,+ mock: meta.mock,+ abort: meta.abort.asSignal(),+ schema: exchangeScrapeResponseSchema,+ });++ if (!response.success) {+ logger.warn("Exchange scrape failed", {+ ...requestLogContext,+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId,+ errorCode: response.error?.code,+ durationMs: Date.now() - startTime,+ });+ throw new EngineError("Exchange request failed");+ }++ const responseLogContext = getExchangeResponseLogContext(+ response.data.metadata,+ );++ logger.info("Exchange scrape completed", {+ ...requestLogContext,+ ...responseLogContext,+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId,+ integrationId: response.data.source?.provider,+ accessEventId: response.accessEventId,+ creditsCost: response.creditsCost,+ durationMs: Date.now() - startTime,+ });++ setSpanAttributes(span, {+ "exchange.integration_id": response.data.source?.provider,+ "exchange.credits_cost": response.creditsCost,+ "exchange.cache_state": responseLogContext.cacheState,+ "exchange.cache_age_ms": responseLogContext.cacheAgeMs,+ "exchange.duration_ms": Date.now() - startTime,+ });++ return {+ url: response.data.url ?? url,+ html: buildMetadataHtml(response.data.title, response.data.description),+ markdown: response.data.markdown,+ json: response.data.json,+ statusCode: 200,+ contentType: "text/markdown",+ proxyUsed: "basic",+ exchange: {+ handled: true,+ creditsCost: response.creditsCost,+ ...(response.accessEventId === undefined+ ? {}+ : { accessEventId: response.accessEventId }),+ ...(response.data.source?.provider === undefined+ ? {}+ : { integrationId: response.data.source.provider }),+ },+ };+ } catch (error) {+ logger.warn("Exchange scrape errored", {+ ...requestLogContext,+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId,+ durationMs: Date.now() - startTime,+ errorMessage: error instanceof Error ? error.message : String(error),+ error,+ });+ throw error;+ }+ });+}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 8a9b2f727..60b986bc7 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -7,6 +7,10 @@ import {scrapeURLWithFireEngineChromeCDP,scrapeURLWithFireEngineTLSClient,} from "./fire-engine";+import {+ exchangeMaxReasonableTime,+ scrapeURLWithExchange,+} from "./exchange";import { pdfMaxReasonableTime, scrapePDF } from "./pdf";import { fetchMaxReasonableTime, scrapeURLWithFetch } from "./fetch";import {@@ -30,8 +34,14 @@ import { getPDFMaxPages } from "../../../controllers/v2/types";import type { PdfMetadata } from "./pdf/types";import { BrandingProfile } from "../../../types/branding";import { BrandingNotSupportedError } from "../error";+import { isUrlBlocked } from "../../WebScraper/utils/blocklist";+import {+ canUseExchangeForRequest,+ type ExchangeScrapeMetadata,+} from "../../../lib/exchange";export type Engine =+ | "exchange"| "fire-engine;chrome-cdp"| "fire-engine(retry);chrome-cdp"| "fire-engine;chrome-cdp;stealth"@@ -162,11 +172,13 @@ export type EngineScrapeResult = {proxyUsed: "basic" | "stealth";timezone?: string;+ exchange?: ExchangeScrapeMetadata;};const engineHandlers: {[E in Engine]: (meta: Meta) => Promise<EngineScrapeResult>;} = {+ "exchange": scrapeURLWithExchange,index: scrapeURLWithIndex,"index;documents": scrapeURLWithIndex,"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,@@ -186,6 +198,7 @@ const engineHandlers: {const engineMRTs: {[E in Engine]: (meta: Meta) => number;} = {+ "exchange": exchangeMaxReasonableTime,index: indexMaxReasonableTime,"index;documents": indexMaxReasonableTime,"fire-engine;chrome-cdp": meta =>@@ -218,6 +231,27 @@ const engineOptions: {quality: number;};} = {+ "exchange": {+ features: {+ actions: false,+ waitFor: false,+ screenshot: false,+ "screenshot@fullScreen": false,+ pdf: false,+ document: false,+ audio: false,+ video: false,+ atsv: false,+ location: false,+ mobile: false,+ skipTlsVerification: true,+ useFastMode: true,+ stealthProxy: false,+ branding: false,+ disableAdblock: false,+ },+ quality: 2000,+ },index: {features: {actions: false,@@ -547,6 +581,73 @@ export async function buildFallbackList(meta: Meta): Promise<unsupportedFeatures: Set<FeatureFlag>;}[]> {+ if (+ !meta.internalOptions.agentIndexOnly &&+ meta.internalOptions.forceEngine === undefined+ ) {+ if (+ await canUseExchangeForRequest({+ url: meta.rewrittenUrl ?? meta.url,+ formats: meta.options.formats,+ actions: meta.options.actions,+ headers: meta.options.headers,+ waitFor: meta.options.waitFor,+ mobile: meta.options.mobile,+ location: meta.options.location,+ proxy: meta.options.proxy,+ blockAds: meta.options.blockAds,+ profile: meta.options.profile,+ atsv: meta.internalOptions.atsv,+ minAge: meta.options.minAge,+ includeTags: meta.options.includeTags,+ excludeTags: meta.options.excludeTags,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,+ lockdown: meta.options.lockdown,+ flags: meta.internalOptions.teamFlags ?? null,+ })+ ) {+ return [+ {+ engine: "exchange",+ unsupportedFeatures: new Set(),+ },+ ];+ }++ // A blocked URL can only have been admitted by the Exchange bypass in+ // blocklistMiddleware, which only applies to flagged orgs; if the+ // Exchange is no longer usable by execution time (catalog changed,+ // service down), fail closed rather than letting normal engines scrape+ // a blocklisted site. An error here also fails closed: this branch only+ // runs for flagged orgs, and a retryable scrape failure is preferable+ // to scraping a potentially blocklisted site with normal engines.+ if (+ meta.internalOptions.teamFlags?.professionalProfileCompanyDataBeta ===+ true+ ) {+ try {+ if (+ isUrlBlocked(+ meta.rewrittenUrl ?? meta.url,+ meta.internalOptions.teamFlags ?? null,+ {+ team_id: meta.internalOptions.teamId ?? null,+ origin: null,+ },+ )+ ) {+ return [];+ }+ } catch (error) {+ meta.logger.warn(+ "Exchange blocklist re-check failed; failing closed",+ { error },+ );+ return [];+ }+ }+ }+const shouldPrioritizeTlsClient = meta.options.__experimental_engpicker? (await queryEngpickerVerdict(meta.options.__experimental_omceDomain ?? new URL(meta.url).hostname,diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 53012521b..3863f972e 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -52,6 +52,9 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {!meta.internalOptions.zeroDataRetention &&meta.winnerEngine !== "index" &&meta.winnerEngine !== "index;documents" &&+ // Exchange-delivered content is never stored on the Firecrawl side:+ // every access must go through the Exchange and its ledger.+ meta.winnerEngine !== "exchange" &&!(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers)) &&!meta.options.parsers?.some(parser => {if (diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex fb1e738f6..5357e6167 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -87,6 +87,7 @@ import { writeFile } from "node:fs/promises";import path from "node:path";import { tmpdir } from "node:os";import { randomUUID } from "node:crypto";+import type { ExchangeScrapeMetadata } from "../../lib/exchange";import {checkUrl,type ThreatCheckDedup,@@ -101,6 +102,7 @@ export type ScrapeUrlResponse =success: true;document: Document;unsupportedFeatures?: Set<FeatureFlag>;+ exchange?: ExchangeScrapeMetadata;/*** Threat protection decisions made for this scrape (initial domain* check + any redirect re-checks, in order). Read by the billing layer@@ -1071,6 +1073,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {success: true,document,unsupportedFeatures: result.unsupportedFeatures,+ exchange: engineResult.exchange,};});}diff --git a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.tsindex 7b350ea42..41a626e68 100644--- a/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts@@ -27,6 +27,11 @@ function shouldIndexForSearch(meta: Meta, document: Document): boolean {return false;}+ // Exchange-delivered content is never stored on the Firecrawl side.+ if (meta.winnerEngine === "exchange") {+ return false;+ }+// Lockdown must not forward the target URL to any external service.if (meta.options.lockdown) {return false;diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex f3d968e02..41d0b7b20 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -99,6 +99,11 @@ import {recordMonitorScrapeFailure,recordMonitorScrapeSuccess,} from "../monitoring/results";+import {+ reportExchangeBilling,+ warmExchangeCatalog,+ type ExchangeScrapeMetadata,+} from "../../lib/exchange";configDotenv();@@ -108,6 +113,7 @@ const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;if (require.main === module) {cacheableLookup.install(http.globalAgent);cacheableLookup.install(https.globalAgent);+ warmExchangeCatalog();}async function billScrapeJob(@@ -118,6 +124,7 @@ async function billScrapeJob(flags: TeamFlags,error?: Error | null,unsupportedFeatures?: Set<FeatureFlag>,+ exchange?: ExchangeScrapeMetadata,threatDecisions?: ThreatDecision[],) {let creditsToBeBilled: number | null = null;@@ -146,6 +153,7 @@ async function billScrapeJob(flags,error,unsupportedFeatures,+ exchange,threatDecisions,);@@ -196,6 +204,18 @@ async function billScrapeJob(},);+ // Reconcile the Exchange ledger. Awaited so the confirmation lands+ // before any later failure path could attempt to void this access -+ // the service rejects confirmed->void, making the order decisive.+ // reportExchangeBilling never throws and is bounded by its timeout.+ if (exchange?.accessEventId !== undefined) {+ await reportExchangeBilling({+ accessEventId: exchange.accessEventId,+ status: "confirmed",+ billingReference: billingJobId,+ });+ }+return creditsToBeBilled;} catch (error) {logger.error(@@ -663,6 +683,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {(await getACUCTeam(job.data.team_id))?.flags ?? null,undefined,pipeline.unsupportedFeatures,+ pipeline.exchange,pipeline.threatDecisions,);@@ -754,6 +775,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {(await getACUCTeam(job.data.team_id))?.flags ?? null,undefined,pipeline.unsupportedFeatures,+ pipeline.exchange,pipeline.threatDecisions,);@@ -954,9 +976,23 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {(await getACUCTeam(job.data.team_id))?.flags ?? null,error instanceof Error ? error : null,undefined,+ undefined,pipeline?.threatDecisions,);+ // The Exchange delivered this access but the scrape ultimately failed,+ // so the customer was never billed for it - void the access event so+ // the Exchange ledger reconciles. Fire-and-forget. If billing did+ // happen before a later step failed, its confirmation was awaited+ // above and the service rejects confirmed->void, so a paid access can+ // never be marked void.+ if (pipeline?.success && pipeline.exchange?.accessEventId !== undefined) {+ void reportExchangeBilling({+ accessEventId: pipeline.exchange.accessEventId,+ status: "void",+ });+ }+logger.debug("Logging job to DB...");await logScrape({