Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): route-independent Exchange terms handling and LinkedIn enrichment routing (#4846)
Gergő Móricz33c042fd1e3c
diff --git a/apps/api/src/__tests__/snips/v2/crawl-errors-terms.test.ts b/apps/api/src/__tests__/snips/v2/crawl-errors-terms.test.tsnew file mode 100644index 000000000..48e050bf5--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/crawl-errors-terms.test.ts@@ -0,0 +1,88 @@+/**+ * Unit test for the crawl/batch errors listing: a page that failed on+ * unaccepted provider terms carries the same `requiresAction` a single+ * scrape's 403 body does, on v1 and v2.+ */++const mocks = vi.hoisted(() => ({+ getCrawl: vi.fn(),+ getGroupJobs: vi.fn(),+ smembers: vi.fn(),+}));++vi.mock("../../../lib/crawl-redis", () => ({+ getCrawl: mocks.getCrawl,+}));+vi.mock("../../../services/worker/nuq-router", () => ({+ scrapeQueue: { getGroupJobs: mocks.getGroupJobs },+}));+vi.mock("../../../services/redis", () => ({+ redisEvictConnection: { smembers: mocks.smembers },+}));++import { crawlErrorsController as v1CrawlErrorsController } from "../../../controllers/v1/crawl-errors";+import { crawlErrorsController as v2CrawlErrorsController } from "../../../controllers/v2/crawl-errors";+import { ThirdPartyDataTermsRequiredError } from "../../../lib/exchange";+import { serializeTransportableError } from "../../../lib/error-serde";+import { SiteError } from "../../../scraper/scrapeURL/error";++describe.each([+ ["v1", v1CrawlErrorsController],+ ["v2", v2CrawlErrorsController],+])("%s crawl errors", (_version, crawlErrorsController) => {+ const TERMS = { key: "acme", version: "2026-01-01" };++ beforeEach(() => {+ mocks.getCrawl.mockResolvedValue({ team_id: "team-1" });+ mocks.smembers.mockResolvedValue([]);+ mocks.getGroupJobs.mockResolvedValue([+ {+ id: "job-terms",+ data: { url: "https://profiles.example/person/example-person" },+ failedReason: serializeTransportableError(+ new ThirdPartyDataTermsRequiredError(TERMS),+ ),+ },+ {+ id: "job-site",+ data: { url: "https://down.example/" },+ failedReason: serializeTransportableError(+ new SiteError("ERR_CONNECTION_RESET"),+ ),+ },+ ]);+ });++ async function listErrors() {+ let body: any;+ const res: any = {+ status: () => res,+ json: (payload: unknown) => {+ body = payload;+ return res;+ },+ };+ await crawlErrorsController(+ {+ params: { jobId: "crawl-1" },+ auth: { team_id: "team-1" },+ } as any,+ res,+ );+ return body;+ }++ it("carries requiresAction on terms-required entries only", async () => {+ const { errors } = await listErrors();+ const expected = new ThirdPartyDataTermsRequiredError(TERMS).response();++ expect(errors.find((e: any) => e.id === "job-terms")).toMatchObject({+ code: "THIRD_PARTY_DATA_TERMS_REQUIRED",+ error: expected.error,+ requiresAction: expected.requiresAction,+ });+ expect(errors.find((e: any) => e.id === "job-site")).not.toHaveProperty(+ "requiresAction",+ );+ });+});diff --git a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts b/apps/api/src/__tests__/snips/v2/exchange-routing.test.tsnew file mode 100644index 000000000..531c99840--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts@@ -0,0 +1,295 @@+/**+ * Unit tests for Exchange routing: the scrape blocklist gate, engine+ * selection in buildFallbackList, and how the exchange engine reports the+ * Exchange's refusals. Unaccepted terms only surface as an error when they+ * are all that keeps the Exchange from serving a blocklisted URL; any other+ * URL scrapes normally, the same way on every route.+ */++const mocks = vi.hoisted(() => ({+ isUrlBlocked: vi.fn(+ (url: string) => new URL(url).hostname === "profiles.example",+ ),+ robustFetch: vi.fn(),+ exchangeRequest: vi.fn(),+}));++vi.mock("../../../scraper/WebScraper/utils/blocklist", () => ({+ isUrlBlocked: mocks.isUrlBlocked,+}));+vi.mock("../../../scraper/scrapeURL/lib/fetch", () => ({+ robustFetch: mocks.robustFetch,+}));+vi.mock("../../../services/alexandria/client", () => ({+ exchangeRequest: mocks.exchangeRequest,+}));++describe("Exchange routing", () => {+ let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;+ let scrapeURLWithExchange: typeof import("../../../scraper/scrapeURL/engines/exchange.js").scrapeURLWithExchange;+ let scrapeBlocklistMiddleware: typeof import("../../../routes/shared.js").scrapeBlocklistMiddleware;+ let exchange: typeof import("../../../lib/exchange.js");+ let scrapeErrors: typeof import("../../../scraper/scrapeURL/error.js");+ let config: typeof import("../../../config.js").config;+ let originalUseDbAuthentication: boolean | undefined;++ const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;+ const originalExchangeUrl = process.env.FIRE_EXCHANGE_URL;++ const TERMS = { key: "acme", version: "2026-01-01" };+ const BLOCKED_URL = "https://profiles.example/person/example-person";+ const OPEN_URL = "https://open.example/person/example-person";++ const ACCEPTED_FLAGS = {+ professionalProfileCompanyDataBeta: true,+ organizationDataSourceAccess: {+ acme: {+ status: "enabled",+ termsKey: TERMS.key,+ termsVersion: TERMS.version,+ },+ },+ };+ const UNACCEPTED_FLAGS = { professionalProfileCompanyDataBeta: true };++ beforeAll(async () => {+ process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine";+ process.env.FIRE_EXCHANGE_URL = "http://test-exchange";++ // Re-import so modules read the env vars set above at eval time.+ vi.resetModules();+ ({ buildFallbackList } = await import(+ "../../../scraper/scrapeURL/engines/index.js"+ ));+ ({ scrapeURLWithExchange } = await import(+ "../../../scraper/scrapeURL/engines/exchange.js"+ ));+ ({ scrapeBlocklistMiddleware } = await import("../../../routes/shared.js"));+ exchange = await import("../../../lib/exchange.js");+ scrapeErrors = await import("../../../scraper/scrapeURL/error.js");+ ({ config } = await import("../../../config.js"));+ });++ beforeEach(() => {+ originalUseDbAuthentication = config.USE_DB_AUTHENTICATION;+ config.USE_DB_AUTHENTICATION = true;+ // The ledger is unreachable: only the flags can show acceptance.+ mocks.exchangeRequest.mockRejectedValue(new Error("offline"));+ mocks.robustFetch.mockReset();+ exchange.setExchangeProvidersForTest([+ {+ id: "acme",+ creditsCost: 12,+ terms: TERMS,+ routes: [{ domains: ["profiles.example", "open.example"] }],+ },+ ]);+ });++ afterEach(() => {+ config.USE_DB_AUTHENTICATION = originalUseDbAuthentication;+ exchange.clearExchangeProvidersForTest();+ });++ afterAll(() => {+ if (originalFireEngineUrl === undefined) {+ delete process.env.FIRE_ENGINE_BETA_URL;+ } else {+ process.env.FIRE_ENGINE_BETA_URL = originalFireEngineUrl;+ }+ if (originalExchangeUrl === undefined) {+ delete process.env.FIRE_EXCHANGE_URL;+ } else {+ process.env.FIRE_EXCHANGE_URL = originalExchangeUrl;+ }+ });++ const buildStubMeta = (url: string, teamFlags: Record<string, unknown>) =>+ ({+ id: "test",+ url,+ options: {+ formats: [{ type: "markdown" }],+ maxAge: 3600000,+ },+ internalOptions: { teamId: "team-1", orgId: "org-1", teamFlags },+ featureFlags: new Set(),+ mock: null,+ abort: { asSignal: () => undefined },+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ debug: vi.fn(),+ error: vi.fn(),+ child: vi.fn().mockReturnThis(),+ },+ }) as any;++ // Runs the scrape blocklist gate and resolves with the response it sent,+ // or `next` when it let the request through.+ function runGate(+ body: Record<string, unknown>,+ flags: Record<string, unknown>,+ ): Promise<{ next: true } | { status: number; body: any }> {+ return new Promise((resolve, reject) => {+ let statusCode = 200;+ const res: any = {+ headersSent: false,+ status(code: number) {+ statusCode = code;+ return res;+ },+ json(payload: unknown) {+ resolve({ status: statusCode, body: payload });+ return res;+ },+ };+ scrapeBlocklistMiddleware(+ { body, acuc: { flags, team_id: "team-1", org_id: "org-1" } } as any,+ res,+ (error?: unknown) => (error ? reject(error) : resolve({ next: true })),+ );+ });+ }++ describe("scrape blocklist gate", () => {+ it("lets an unblocked URL through even when its provider's terms are unaccepted", async () => {+ await expect(+ runGate({ url: OPEN_URL, formats: ["markdown"] }, UNACCEPTED_FLAGS),+ ).resolves.toEqual({ next: true });+ });++ it("asks for the terms when they are all that keeps a blocked URL out", async () => {+ const result = await runGate(+ { url: BLOCKED_URL, formats: ["markdown"] },+ UNACCEPTED_FLAGS,+ );++ expect(result).toEqual({+ status: 403,+ body: new exchange.ThirdPartyDataTermsRequiredError(TERMS).response(),+ });+ });++ it("lets a blocked URL through when the Exchange can serve it", async () => {+ await expect(+ runGate({ url: BLOCKED_URL, formats: ["markdown"] }, ACCEPTED_FLAGS),+ ).resolves.toEqual({ next: true });+ });++ it("blocks a blocked URL the Exchange cannot serve", async () => {+ const result = await runGate(+ {+ url: BLOCKED_URL,+ formats: ["markdown"],+ actions: [{ type: "wait", milliseconds: 1000 }],+ },+ ACCEPTED_FLAGS,+ );++ expect(result).toMatchObject({+ status: 403,+ body: { success: false },+ });+ expect((result as any).body.code).toBeUndefined();+ });+ });++ describe("buildFallbackList", () => {+ it("routes to the Exchange when the terms are accepted", async () => {+ const fallback = await buildFallbackList(+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),+ );++ expect(fallback.map(f => f.engine)).toEqual(["exchange"]);+ });++ it("scrapes an unblocked URL normally when the terms are unaccepted", async () => {+ const fallback = await buildFallbackList(+ buildStubMeta(OPEN_URL, UNACCEPTED_FLAGS),+ );+ const engines = fallback.map(f => f.engine);++ expect(engines).not.toContain("exchange");+ expect(engines.length).toBeGreaterThan(0);+ });++ it("raises the terms error for a blocked URL when the terms are unaccepted", async () => {+ const error = await buildFallbackList(+ buildStubMeta(BLOCKED_URL, UNACCEPTED_FLAGS),+ ).catch(e => e);++ expect(error).toBeInstanceOf(exchange.ThirdPartyDataTermsRequiredError);+ expect(error.response()).toEqual(+ new exchange.ThirdPartyDataTermsRequiredError(TERMS).response(),+ );+ });++ it("fails closed for a blocked URL the Exchange cannot serve", async () => {+ const meta = buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS);+ meta.options.actions = [{ type: "wait", milliseconds: 1000 }];++ await expect(buildFallbackList(meta)).resolves.toEqual([]);+ });+ });++ describe("exchange engine", () => {+ const failure = (code: string, message: string) => ({+ success: false,+ error: { code, message, retryable: false },+ });++ it("reports a missing record as THIRD_PARTY_DATA_NOT_FOUND", async () => {+ mocks.robustFetch.mockResolvedValue(+ failure(+ "record_not_found",+ "The requested Exchange record was not found.",+ ),+ );++ const error = await scrapeURLWithExchange(+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),+ ).catch(e => e);++ expect(error).toBeInstanceOf(scrapeErrors.ExchangeRefusedError);+ expect(error.code).toBe("THIRD_PARTY_DATA_NOT_FOUND");+ expect(error.message).toBe(+ "The requested Exchange record was not found.",+ );+ });++ it("reports a provider the team lacks as THIRD_PARTY_DATA_NOT_ENABLED", async () => {+ mocks.robustFetch.mockResolvedValue(+ failure(+ "provider_not_enabled",+ "Exchange provider acme is not enabled for this team.",+ ),+ );++ const error = await scrapeURLWithExchange(+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),+ ).catch(e => e);++ expect(error).toBeInstanceOf(scrapeErrors.ExchangeRefusedError);+ expect(error.code).toBe("THIRD_PARTY_DATA_NOT_ENABLED");+ expect(error.message).toBe(+ "Exchange provider acme is not enabled for this team.",+ );+ });++ it("keeps any other failure an engine error", async () => {+ mocks.robustFetch.mockResolvedValue(+ failure(+ "internal_error",+ "The Exchange request could not be completed.",+ ),+ );++ const error = await scrapeURLWithExchange(+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),+ ).catch(e => e);++ expect(error).toBeInstanceOf(scrapeErrors.EngineError);+ });+ });+});diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex 9706c3d30..808f7646a 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -12,6 +12,7 @@ import { getCrawlJobAccess } from "../../lib/operational-job-access";import { logger as _logger } from "../../lib/logger";import { deserializeTransportableError } from "../../lib/error-serde";import { TransportableError } from "../../lib/error";+import { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";import { scrapeQueue } from "../../services/worker/nuq-router";configDotenv();@@ -87,6 +88,9 @@ export async function crawlErrorsController(? {code: error.code,error: error.message,+ ...(error instanceof ThirdPartyDataTermsRequiredError+ ? { requiresAction: error.requiresAction }+ : {}),}: {error: x.failedReason!,diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 5aa14a33d..8e4d30d77 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -12,6 +12,7 @@ import { v7 as uuidv7 } from "uuid";import { getJobPriority } from "../../lib/job-priority";import { fromV1ScrapeOptions } from "../v2/types";import { TransportableError } from "../../lib/error";+import { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";import { NuQJob } from "../../services/worker/nuq";import { checkPermissions } from "../../lib/permissions";import {@@ -410,6 +411,29 @@ async function scrapeControllerInner(});}+ if (e instanceof ThirdPartyDataTermsRequiredError) {+ return res.status(403).json(e.response());+ }++ if (+ e.code === "THIRD_PARTY_DATA_NOT_ENABLED" ||+ e.code === "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED"+ ) {+ return res.status(403).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }++ if (e.code === "THIRD_PARTY_DATA_NOT_FOUND") {+ return res.status(404).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+return res.status(timeoutErr ? 408 : 500).json({success: false,code: e.code,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 52d617079..f80d7a0c1 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1,4 +1,5 @@import type { AgentInteropStatus } from "../../lib/agent-interop";+import type { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";import { Request } from "express";import { config } from "../../config";import { z } from "zod";@@ -1303,6 +1304,7 @@ export type CrawlErrorsResponse =timestamp?: string;url: string;error: string;+ requiresAction?: ThirdPartyDataTermsRequiredError["requiresAction"];}[];robotsBlocked: string[];};diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsindex 9706c3d30..808f7646a 100644--- a/apps/api/src/controllers/v2/crawl-errors.ts+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -12,6 +12,7 @@ import { getCrawlJobAccess } from "../../lib/operational-job-access";import { logger as _logger } from "../../lib/logger";import { deserializeTransportableError } from "../../lib/error-serde";import { TransportableError } from "../../lib/error";+import { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";import { scrapeQueue } from "../../services/worker/nuq-router";configDotenv();@@ -87,6 +88,9 @@ export async function crawlErrorsController(? {code: error.code,error: error.message,+ ...(error instanceof ThirdPartyDataTermsRequiredError+ ? { requiresAction: error.requiresAction }+ : {}),}: {error: x.failedReason!,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 83af538d8..3884b47f7 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -17,6 +17,7 @@ import {getTimeoutProcessingDetails,TransportableError,} from "../../lib/error";+import { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";import { NuQJob } from "../../services/worker/nuq";import { checkPermissions } from "../../lib/permissions";import {@@ -593,6 +594,38 @@ export async function scrapeController(});}+ if (e instanceof ThirdPartyDataTermsRequiredError) {+ setSpanAttributes(span, {+ "scrape.status_code": 403,+ });+ return res.status(403).json(e.response());+ }++ if (+ e.code === "THIRD_PARTY_DATA_NOT_ENABLED" ||+ e.code === "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED"+ ) {+ setSpanAttributes(span, {+ "scrape.status_code": 403,+ });+ return res.status(403).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }++ if (e.code === "THIRD_PARTY_DATA_NOT_FOUND") {+ setSpanAttributes(span, {+ "scrape.status_code": 404,+ });+ return res.status(404).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+const statusCode = timeoutErr ? 408 : 500;setSpanAttributes(span, {"scrape.status_code": statusCode,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 3a958a0e5..20ce7703a 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1,4 +1,5 @@import type { AgentInteropStatus } from "../../lib/agent-interop";+import type { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";import { Request, Response } from "express";import { hasCategory } from "../../lib/search-query-builder";import { config } from "../../config";@@ -2013,6 +2014,7 @@ export type CrawlErrorsResponse =url: string;code?: ErrorCodes;error: string;+ requiresAction?: ThirdPartyDataTermsRequiredError["requiresAction"];}[];robotsBlocked: string[];};diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 6033b4a25..556f58e09 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -42,13 +42,17 @@ import {PromptInjectionDetectedError,JsonExtractionContentTooLargeError,XTwitterConfigurationError,+ ExchangeRefusedError,} from "../scraper/scrapeURL/error";import { UnsafeDomainBlockedError } from "./threat-protection/error";+import { ThirdPartyDataTermsRequiredError } from "./exchange";// TODO: figure out correct typing for thisconst errorMap: Record<ErrorCodes, any> = {- // Terms responses are API-level, never transported through workers.- THIRD_PARTY_DATA_TERMS_REQUIRED: null,+ THIRD_PARTY_DATA_TERMS_REQUIRED: ThirdPartyDataTermsRequiredError,+ THIRD_PARTY_DATA_NOT_FOUND: ExchangeRefusedError,+ THIRD_PARTY_DATA_NOT_ENABLED: ExchangeRefusedError,+ THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED: ExchangeRefusedError,SAFE_MODE_BLOCKED: null,SCRAPE_SITE_RESTRICTION_BLOCKED: SiteRestrictionError,SCRAPE_TIMEOUT: ScrapeJobTimeoutError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 09394ba0b..b32d40f53 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -2,6 +2,9 @@ import { UNSUPPORTED_SITE_MESSAGE } from "./strings";export type ErrorCodes =| "THIRD_PARTY_DATA_TERMS_REQUIRED"+ | "THIRD_PARTY_DATA_NOT_FOUND"+ | "THIRD_PARTY_DATA_NOT_ENABLED"+ | "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED"| "SCRAPE_TIMEOUT"| "MAP_TIMEOUT"| "UNKNOWN_ERROR"diff --git a/apps/api/src/lib/exchange-request.ts b/apps/api/src/lib/exchange-request.tsindex 17238155b..9c77bf952 100644--- a/apps/api/src/lib/exchange-request.ts+++ b/apps/api/src/lib/exchange-request.ts@@ -1,11 +1,6 @@-import {- getExchangeAccessForRequest,- type ExchangeAccess,-} from "./exchange";+import { getExchangeAccessForRequest, type ExchangeAccess } from "./exchange";-type ExchangeFlags = Parameters<- typeof getExchangeAccessForRequest->[0]["flags"];+type ExchangeFlags = Parameters<typeof getExchangeAccessForRequest>[0]["flags"];/*** Run the Exchange access gate against an API request body, sourcing the@@ -17,7 +12,10 @@ export function getExchangeAccessForRequestBody(input: {body: Record<string, any>;flags: ExchangeFlags;url: string;+ blocked: boolean;zeroDataRetention: boolean;+ teamId: string | null;+ orgId: string | null;}): Promise<ExchangeAccess> {const body = input.body ?? {};const scrapeOptions =@@ -27,6 +25,9 @@ export function getExchangeAccessForRequestBody(input: {return getExchangeAccessForRequest({url: input.url,+ teamId: input.teamId,+ orgId: input.orgId,+ blocked: input.blocked,formats: scrapeOptions.formats,actions: scrapeOptions.actions,headers: scrapeOptions.headers,diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.tsindex cfdd098bc..92aee817f 100644--- a/apps/api/src/lib/exchange.test.ts+++ b/apps/api/src/lib/exchange.test.ts@@ -2,6 +2,11 @@ import { fetch } from "undici";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { config } from "../config";+import { exchangeRequest } from "../services/alexandria/client";+import {+ deserializeTransportableError,+ serializeTransportableError,+} from "./error-serde";import {canUseExchangeForRequest,clearExchangeProvidersForTest,@@ -10,17 +15,21 @@ import {getExchangeRequestLogContext,getExchangeResponseLogContext,getExchangeSuccessCredits,- getThirdPartyDataTermsRequiredResponse,isSuccessfulExchangeStatusCode,isSupportedExchangeFormatRequest,resolveExchangeProvider,setExchangeProvidersForTest,+ ThirdPartyDataTermsRequiredError,} from "./exchange";vi.mock("undici", () => ({fetch: vi.fn(),}));+vi.mock("../services/alexandria/client", () => ({+ exchangeRequest: vi.fn(),+}));+const originalConfig = {FIRE_EXCHANGE_URL: config.FIRE_EXCHANGE_URL,USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,@@ -290,7 +299,7 @@ describe("Exchange routing", () => {expect(getExchangeResponseLogContext(null)).toEqual({});});- it("accepts only formats the Exchange can return directly", () => {+ it("accepts only formats derivable from the Exchange's markdown", () => {expect(isSupportedExchangeFormatRequest(undefined)).toBe(true);expect(isSupportedExchangeFormatRequest([{ type: "markdown" }])).toBe(true);expect(isSupportedExchangeFormatRequest(["json"])).toBe(true);@@ -300,7 +309,10 @@ describe("Exchange routing", () => {{ type: "json" },]),).toBe(true);- expect(isSupportedExchangeFormatRequest([{ type: "html" }])).toBe(false);+ expect(isSupportedExchangeFormatRequest([{ type: "html" }])).toBe(true);+ expect(isSupportedExchangeFormatRequest([{ type: "screenshot" }])).toBe(+ false,+ );// deterministicJson extractors run against page HTML, which Exchange// responses do not carry.expect(@@ -401,7 +413,9 @@ describe("Exchange routing", () => {terms: ACME_TERMS,});- const response = getThirdPartyDataTermsRequiredResponse(ACME_TERMS);+ const response = new ThirdPartyDataTermsRequiredError(+ ACME_TERMS,+ ).response();expect(response).toMatchObject({success: false,code: "THIRD_PARTY_DATA_TERMS_REQUIRED",@@ -485,7 +499,7 @@ describe("Exchange routing", () => {).resolves.toEqual({ allowed: false, termsRequired: false });});- it("does not route unless the beta flag is enabled", async () => {+ it("requires the beta flag except for blocked URLs", async () => {await expect(canUseExchangeForRequest({url: "https://profiles.example/person/example-person",@@ -500,6 +514,27 @@ describe("Exchange routing", () => {flags: { professionalProfileCompanyDataBeta: false },}),).resolves.toBe(false);++ // A blocked URL has no other way to be served, and still needs its terms.+ await expect(+ canUseExchangeForRequest({+ url: "https://facts.example/records/1",+ formats: [{ type: "markdown" }],+ waitFor: 1000,+ blocked: true,+ }),+ ).resolves.toBe(true);+ await expect(+ getExchangeAccessForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ blocked: true,+ }),+ ).resolves.toEqual({+ allowed: false,+ termsRequired: true,+ terms: ACME_TERMS,+ });});it("does not route unless the Exchange is configured", async () => {@@ -670,3 +705,264 @@ describe("Exchange routing", () => {expect(fetch).toHaveBeenCalledTimes(1);});});++describe("Exchange terms acceptance", () => {+ const DIGEST = "a".repeat(64);+ const PROFILE_REQUEST = {+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ teamId: "team-1",+ orgId: "org-1",+ };++ // Answers the Exchange's requirements and ledger status routes the way+ // the acceptance fallback reads them.+ function ledgerAnswers(+ ledger: {+ version: string;+ textHash: string;+ acceptedAt?: string;+ }[],+ ) {+ vi.mocked(exchangeRequest).mockImplementation(async ({ path }) => {+ if (path === "/v1/provider-terms/requirements") {+ return {+ status: 200,+ body: {+ providers: [+ {+ provider: "acme",+ required: true,+ terms: { ...ACME_TERMS, digest: DIGEST },+ },+ ],+ },+ };+ }+ if (path.startsWith("/v1/provider-terms/status")) {+ return {+ status: 200,+ body: {+ providers: ledger.map(entry => ({+ provider: "acme",+ revoked: false,+ acceptedAt: "2026-01-05T00:00:00.000Z",+ ...entry,+ })),+ },+ };+ }+ return { status: 404, body: {} };+ });+ }++ beforeEach(() => {+ vi.mocked(exchangeRequest).mockReset();+ config.FIRE_EXCHANGE_URL = "https://exchange.example";+ config.USE_DB_AUTHENTICATION = true;+ setExchangeProvidersForTest(TEST_PROVIDERS);+ });++ afterEach(() => {+ config.FIRE_EXCHANGE_URL = originalConfig.FIRE_EXCHANGE_URL;+ config.USE_DB_AUTHENTICATION = originalConfig.USE_DB_AUTHENTICATION;+ clearExchangeProvidersForTest();+ });++ it("routes when the terms were accepted on the Exchange ledger", async () => {+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: DIGEST }]);++ const access = await getExchangeAccessForRequest({+ ...PROFILE_REQUEST,+ flags: { professionalProfileCompanyDataBeta: true },+ });++ expect(access.allowed).toBe(true);+ });++ it("does not accept a ledger entry for different terms text", async () => {+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: "b".repeat(64) }]);++ await expect(+ getExchangeAccessForRequest({+ ...PROFILE_REQUEST,+ flags: { professionalProfileCompanyDataBeta: true },+ }),+ ).resolves.toEqual({+ allowed: false,+ termsRequired: true,+ terms: ACME_TERMS,+ });+ });++ it("skips the ledger for requests without an organization", async () => {+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: DIGEST }]);++ const access = await getExchangeAccessForRequest({+ ...PROFILE_REQUEST,+ orgId: null,+ flags: { professionalProfileCompanyDataBeta: true },+ });++ expect(access).toMatchObject({ allowed: false, termsRequired: true });+ expect(exchangeRequest).not.toHaveBeenCalled();+ });++ it("reads as not accepted when the ledger is unreachable", async () => {+ vi.mocked(exchangeRequest).mockRejectedValue(new Error("unreachable"));++ await expect(+ getExchangeAccessForRequest({+ ...PROFILE_REQUEST,+ flags: { professionalProfileCompanyDataBeta: true },+ }),+ ).resolves.toMatchObject({ allowed: false, termsRequired: true });+ });++ it("caches ledger answers per organization and provider", async () => {+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: DIGEST }]);+ const input = {+ ...PROFILE_REQUEST,+ flags: { professionalProfileCompanyDataBeta: true },+ };++ await getExchangeAccessForRequest(input);+ await getExchangeAccessForRequest(input);++ // One requirements read and one ledger read, not two of each.+ expect(exchangeRequest).toHaveBeenCalledTimes(2);+ });++ it("does not share cached ledger answers between teams", async () => {+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: DIGEST }]);+ const flags = { professionalProfileCompanyDataBeta: true };++ await getExchangeAccessForRequest({ ...PROFILE_REQUEST, flags });+ await getExchangeAccessForRequest({+ ...PROFILE_REQUEST,+ teamId: "team-2",+ flags,+ });++ expect(exchangeRequest).toHaveBeenCalledTimes(4);+ });++ it("rechecks the ledger once the catalog carries new terms", async () => {+ const input = {+ ...PROFILE_REQUEST,+ flags: { professionalProfileCompanyDataBeta: true },+ };+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: DIGEST }]);+ await expect(getExchangeAccessForRequest(input)).resolves.toMatchObject({+ allowed: true,+ });++ // The provider publishes new terms; the ledger still holds the old ones.+ const NEW_TERMS = { key: "acme", version: "2026-02-01" };+ setExchangeProvidersForTest(+ TEST_PROVIDERS.map(provider =>+ provider.id === "acme" ? { ...provider, terms: NEW_TERMS } : provider,+ ),+ );+ vi.mocked(exchangeRequest).mockImplementation(async ({ path }) =>+ path === "/v1/provider-terms/requirements"+ ? {+ status: 200,+ body: {+ providers: [+ {+ provider: "acme",+ required: true,+ terms: { ...NEW_TERMS, digest: "c".repeat(64) },+ },+ ],+ },+ }+ : {+ status: 200,+ body: {+ providers: [+ {+ provider: "acme",+ revoked: false,+ version: ACME_TERMS.version,+ textHash: DIGEST,+ acceptedAt: "2026-01-05T00:00:00.000Z",+ },+ ],+ },+ },+ );++ await expect(getExchangeAccessForRequest(input)).resolves.toEqual({+ allowed: false,+ termsRequired: true,+ terms: NEW_TERMS,+ });+ });++ it("asks for terms again after an admin revocation until re-accepted", async () => {+ const revokedFlags = {+ professionalProfileCompanyDataBeta: true,+ organizationDataSourceAccess: {+ acme: {+ status: "disabled",+ termsKey: "acme",+ termsVersion: "2026-01-01",+ disabledAt: "2026-01-10T00:00:00.000Z",+ disabledReason: "revoked_by_organization_admin",+ },+ },+ };++ ledgerAnswers([+ {+ version: ACME_TERMS.version,+ textHash: DIGEST,+ acceptedAt: "2026-01-05T00:00:00.000Z",+ },+ ]);+ await expect(+ getExchangeAccessForRequest({ ...PROFILE_REQUEST, flags: revokedFlags }),+ ).resolves.toEqual({+ allowed: false,+ termsRequired: true,+ terms: ACME_TERMS,+ });++ clearExchangeProvidersForTest();+ setExchangeProvidersForTest(TEST_PROVIDERS);+ ledgerAnswers([+ {+ version: ACME_TERMS.version,+ textHash: DIGEST,+ acceptedAt: "2026-01-11T00:00:00.000Z",+ },+ ]);+ const access = await getExchangeAccessForRequest({+ ...PROFILE_REQUEST,+ flags: revokedFlags,+ });+ expect(access.allowed).toBe(true);+ });++ it("carries the terms-required response through the worker queue", () => {+ const error = deserializeTransportableError(+ serializeTransportableError(+ new ThirdPartyDataTermsRequiredError(ACME_TERMS),+ ),+ );++ expect(error).toBeInstanceOf(ThirdPartyDataTermsRequiredError);+ expect((error as ThirdPartyDataTermsRequiredError).response()).toEqual(+ new ThirdPartyDataTermsRequiredError(ACME_TERMS).response(),+ );+ expect(error?.message).toBe(+ new ThirdPartyDataTermsRequiredError(ACME_TERMS).response().error,+ );+ expect((error as ThirdPartyDataTermsRequiredError).requiresAction).toEqual(+ new ThirdPartyDataTermsRequiredError(ACME_TERMS).response()+ .requiresAction,+ );+ });+});diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.tsindex 937ddd8ac..7048bfdcf 100644--- a/apps/api/src/lib/exchange.ts+++ b/apps/api/src/lib/exchange.ts@@ -3,6 +3,8 @@ import { z } from "zod";import { config } from "../config";import type { FormatObject } from "../controllers/v2/types";+import { hasLedgerAcceptance } from "../services/alexandria/terms";+import { type ErrorCodes, TransportableError } from "./error";import { logger as rootLogger } from "./logger";type OrganizationDataSourceAccessRecord = {@@ -22,6 +24,10 @@ type OrganizationDataSourceAccess = Record<type RouteInput = {url: string;+ teamId?: string | null;+ orgId?: string | null;+ /** The URL is on the team's blocklist, so the Exchange is the only way to serve it. */+ blocked?: boolean;formats?: FormatObject[] | unknown[];actions?: unknown[];headers?: Record<string, unknown>;@@ -65,11 +71,22 @@ type ExchangeProvider = {}[];};-// deterministicJson is deliberately unsupported: its extractor scripts run-// against page HTML, which Exchange responses do not carry.-const SUPPORTED_FORMATS = new Set(["markdown", "json"]);+// Formats the regular transformers derive from the Exchange's markdown (and the+// HTML rendered from it). deterministicJson, screenshots and the like need the+// real page, which the Exchange never fetches.+const SUPPORTED_FORMATS = new Set([+ "markdown",+ "html",+ "rawHtml",+ "links",+ "images",+ "json",+ "summary",+ "question",+ "highlights",+ "query",+]);const EXCHANGE_BETA_FLAG = "professionalProfileCompanyDataBeta";-const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED";const EXCHANGE_PROVIDERS_PATH = "/v1/providers";const EXCHANGE_PROVIDERS_TIMEOUT_MS = 2_000;@@ -238,9 +255,16 @@ function providerMatchesUrl(const host = normalizeHost(parsed.hostname);const pathname = parsed.pathname || "/";+ // "*.example.com" claims every subdomain of example.com, never the apex.+ const wildcards = [...host.matchAll(/\./g)].map(+ dot => `*${host.slice(dot.index)}`,+ );return provider.routes.some(route => {- if (!route.domains.has(host)) {+ if (+ !route.domains.has(host) &&+ !wildcards.some(wildcard => route.domains.has(wildcard))+ ) {return false;}@@ -258,16 +282,33 @@ function providerMatchesUrl(});}+// Organizations kept on the direct FullEnrich path for LinkedIn, which predates+// enrichment preferences: they skip the enrichment provider and resolve to the+// next claimant in the catalog.+const ENRICHMENT_PROVIDER_ID = "firecrawl-enrich";+const LEGACY_FULLENRICH_ORGS = new Set([+ "34a599c6-e6c2-4e6f-b563-0e23bb2552c1",+ "adcef175-be20-4534-96ad-fcd413e3c457",+ "2567598d-d959-44d3-8c4a-19fd467ec66d",+ "709cf0a7-7769-4c7e-a5e1-ad44f38f9c36",+]);+export async function resolveExchangeProvider(inputUrl: string,+ orgId?: string | null,): Promise<ExchangeProvider | null> {const providers = await getExchangeProviders();if (providers === null) {return null;}+ const skipEnrichment = orgId != null && LEGACY_FULLENRICH_ORGS.has(orgId);return (- providers.find(provider => providerMatchesUrl(provider, inputUrl)) ?? null+ providers.find(+ provider =>+ !(skipEnrichment && provider.id === ENRICHMENT_PROVIDER_ID) &&+ providerMatchesUrl(provider, inputUrl),+ ) ?? null);}@@ -356,15 +397,80 @@ export function isSupportedExchangeFormatRequest(type DataSourceAccessDecision = "allowed" | "terms_required" | "not_enabled";-function getProviderAccessDecision(+const LEDGER_ACCEPTANCE_TIMEOUT_MS = 2_000;+const LEDGER_ACCEPTED_TTL_MS = 60_000;+const LEDGER_NOT_ACCEPTED_TTL_MS = 10_000;+const LEDGER_ACCEPTANCE_CACHE_MAX_ENTRIES = 10_000;++const ledgerAcceptanceCache = new Map<+ string,+ { expiresAt: number; value: Promise<boolean> }+>();++// The ledger lookup costs two Exchange calls and sits on the scrape path, so+// answers are cached per process: briefly when not accepted, so a fresh+// acceptance is picked up quickly, and longer once accepted. Keyed by the+// team the Exchange calls are made for and the catalog's terms identity, so+// new terms are rechecked as soon as the catalog carries them.+function getLedgerAcceptance(input: {+ teamId: string;+ orgId: string;+ provider: string;+ terms: ExchangeTerms;+ revocation?: { disabledAt: unknown };+}): Promise<boolean> {+ const key = [+ input.teamId,+ input.orgId,+ input.provider,+ input.terms.key,+ input.terms.version,+ input.revocation === undefined ? "" : String(input.revocation.disabledAt),+ ].join("\0");+ const cached = ledgerAcceptanceCache.get(key);+ if (cached && cached.expiresAt > Date.now()) {+ return cached.value;+ }++ if (ledgerAcceptanceCache.size >= LEDGER_ACCEPTANCE_CACHE_MAX_ENTRIES) {+ ledgerAcceptanceCache.clear();+ }++ const entry = {+ expiresAt: Date.now() + LEDGER_NOT_ACCEPTED_TTL_MS,+ value: Promise.resolve(false),+ };+ entry.value = hasLedgerAcceptance({+ teamId: input.teamId,+ orgId: input.orgId,+ provider: input.provider,+ revocation: input.revocation,+ timeoutMs: LEDGER_ACCEPTANCE_TIMEOUT_MS,+ })+ .catch(() => false)+ .then(accepted => {+ if (accepted) {+ entry.expiresAt = Date.now() + LEDGER_ACCEPTED_TTL_MS;+ }+ return accepted;+ });+ ledgerAcceptanceCache.set(key, entry);+ return entry.value;+}++// Mirrors authorizeProviders (services/alexandria/access.ts), so a provider+// is reachable through Scrape exactly when it is through Alexandria: the+// organizationDataSourceAccess flags first, then the Exchange ledger, where+// the API's accept route records acceptance.+async function getProviderAccessDecision(provider: ExchangeProvider,- flags: RouteInput["flags"],-): DataSourceAccessDecision {+ input: RouteInput,+): Promise<DataSourceAccessDecision> {if (config.USE_DB_AUTHENTICATION !== true) {return "allowed";}- const access = flags?.organizationDataSourceAccess?.[provider.id];+ const access = input.flags?.organizationDataSourceAccess?.[provider.id];const entry = typeof access === "object" && access !== null ? access : null;if (provider.terms === undefined) {@@ -373,22 +479,58 @@ function getProviderAccessDecision(: "allowed";}- if (entry === null) {- return "terms_required";- }+ const teamId = input.teamId ?? null;+ const orgId = input.orgId ?? null;++ if (entry !== null && entry.status !== "enabled") {+ // An admin revocation is lifted by a ledger acceptance recorded after+ // it; until then the provider asks for its terms again.+ const revokedByAdmin =+ entry.status === "disabled" &&+ entry.disabledReason === "revoked_by_organization_admin";+ if (!revokedByAdmin || teamId === null || orgId === null) {+ return "not_enabled";+ }- if (entry.status !== "enabled") {- return "not_enabled";+ return (await getLedgerAcceptance({+ teamId,+ orgId,+ provider: provider.id,+ terms: provider.terms,+ revocation: { disabledAt: entry.disabledAt },+ }))+ ? "allowed"+ : "terms_required";}- return entry.termsKey === provider.terms.key &&+ if (+ entry !== null &&+ entry.termsKey === provider.terms.key &&entry.termsVersion === provider.terms.version- ? "allowed"- : "terms_required";+ ) {+ return "allowed";+ }++ if (+ teamId !== null &&+ orgId !== null &&+ (await getLedgerAcceptance({+ teamId,+ orgId,+ provider: provider.id,+ terms: provider.terms,+ }))+ ) {+ return "allowed";+ }++ return "terms_required";}function isExchangeEligibleRequest(input: RouteInput): boolean {- if (input.flags?.[EXCHANGE_BETA_FLAG] !== true) {+ // Blocked URLs go to the Exchange for every team, since nothing else may+ // serve them; elsewhere it replaces a normal scrape only for the beta.+ if (input.flags?.[EXCHANGE_BETA_FLAG] !== true && input.blocked !== true) {return false;}@@ -408,27 +550,28 @@ function isExchangeEligibleRequest(input: RouteInput): boolean {return false;}- if (input.headers && Object.keys(input.headers).length > 0) {- return false;- }-- if (input.waitFor !== undefined && input.waitFor !== 0) {- return false;- }-- if (input.mobile || input.location || input.blockAds === false) {- return false;- }-// Profile-backed scrapes expect session-specific content, which the// Exchange cannot serve.if (input.profile !== undefined) {return false;}- // atsv is only supported by browser engines; requests that set it keep an- // engine that can honor it instead of routing to the Exchange.- if (input.atsv === true) {+ // Rendering options only mean something for a real page. A blocked URL has+ // no page Firecrawl may render, so they are ignored there; anywhere else a+ // request that sets them keeps the normal engines.+ if (+ input.blocked !== true &&+ ((input.headers !== undefined && Object.keys(input.headers).length > 0) ||+ (input.waitFor !== undefined && input.waitFor !== 0) ||+ input.mobile ||+ input.location ||+ input.blockAds === false ||+ input.atsv === true ||+ input.proxy === "stealth" ||+ input.proxy === "enhanced" ||+ (Array.isArray(input.includeTags) && input.includeTags.length > 0) ||+ (Array.isArray(input.excludeTags) && input.excludeTags.length > 0))+ ) {return false;}@@ -439,18 +582,6 @@ function isExchangeEligibleRequest(input: RouteInput): boolean {return false;}- // Selector-based content filtering does not apply to provider records.- if (- (Array.isArray(input.includeTags) && input.includeTags.length > 0) ||- (Array.isArray(input.excludeTags) && input.excludeTags.length > 0)- ) {- return false;- }-- if (input.proxy === "stealth" || input.proxy === "enhanced") {- return false;- }-if (!isSupportedExchangeFormatRequest(input.formats)) {return false;}@@ -485,12 +616,12 @@ export async function getExchangeAccessForRequest(return { allowed: false, termsRequired: false };}- const provider = await resolveExchangeProvider(input.url);+ const provider = await resolveExchangeProvider(input.url, input.orgId);if (provider === null) {return { allowed: false, termsRequired: false };}- const decision = getProviderAccessDecision(provider, input.flags);+ const decision = await getProviderAccessDecision(provider, input);if (decision === "terms_required" && provider.terms !== undefined) {return { allowed: false, termsRequired: true, terms: provider.terms };}@@ -518,19 +649,70 @@ function getThirdPartyDataTermsUrl(terms: ExchangeTerms): string {return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/alexandria/${encodeURIComponent(terms.key)}`;}-export function getThirdPartyDataTermsRequiredResponse(terms: ExchangeTerms) {- const url = getThirdPartyDataTermsUrl(terms);- return {- success: false as const,- code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE as "THIRD_PARTY_DATA_TERMS_REQUIRED",- error: `An organization admin must accept the ${terms.key} provider's terms (version ${terms.version}) before this request can run. Accept them at ${url}`,- requiresAction: {- type: "accept_terms",- terms: terms.key,- version: terms.version,- url,- },- };+export function getEnrichmentSettingsUrl(): string {+ return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/alexandria?enrichment=true`;+}++/**+ * An organization admin has to accept a provider's terms before the request+ * can run. Transportable, so it crosses the worker queue intact; every+ * surface that reports it sends `response()`, or `requiresAction` alone where+ * the error is one entry of a list.+ */+export class ThirdPartyDataTermsRequiredError extends TransportableError {+ public readonly terms: ExchangeTerms;+ /** Set when the provider is a step in the team's enrichment order, which can drop it instead. */+ public readonly enrichment: boolean;++ constructor(terms: ExchangeTerms, options: { enrichment?: boolean } = {}) {+ const accept = `An organization admin must accept the ${terms.key} provider's terms (version ${terms.version}) before this request can run. Accept them at ${getThirdPartyDataTermsUrl(terms)}`;+ super(+ "THIRD_PARTY_DATA_TERMS_REQUIRED",+ options.enrichment+ ? `${accept}, or turn off ${terms.key} in ${getEnrichmentSettingsUrl()}.`+ : accept,+ );+ this.name = "ThirdPartyDataTermsRequiredError";+ this.terms = { key: terms.key, version: terms.version };+ this.enrichment = options.enrichment === true;+ }++ get requiresAction() {+ return {+ type: "accept_terms" as const,+ terms: this.terms.key,+ version: this.terms.version,+ url: getThirdPartyDataTermsUrl(this.terms),+ };+ }++ response() {+ return {+ success: false as const,+ code: "THIRD_PARTY_DATA_TERMS_REQUIRED" as const,+ error: this.message,+ requiresAction: this.requiresAction,+ };+ }++ serialize() {+ return {+ ...super.serialize(),+ terms: this.terms,+ enrichment: this.enrichment,+ };+ }++ static deserialize(+ _code: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new ThirdPartyDataTermsRequiredError(data.terms, {+ enrichment: data.enrichment,+ });+ x.stack = data.stack;+ return x;+ }}export function getExchangeSuccessCredits(input: {@@ -759,4 +941,5 @@ export function setExchangeProvidersForTest(export function clearExchangeProvidersForTest() {cachedProviders = undefined;providersRequest = undefined;+ ledgerAcceptanceCache.clear();}diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex 6ca688fc0..b9c1e690d 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -38,6 +38,34 @@ describe("calculateCreditsToBeBilled", () => {expect(credits).toBe(12);});+ it("adds format surcharges on top of an Exchange access", async () => {+ const credits = await calculateCreditsToBeBilled(+ {+ formats: [{ type: "json", prompt: "name" }],+ } as any,+ {+ teamId: "team-id",+ orgId: null,+ },+ {+ metadata: {+ statusCode: 200,+ url: "https://profiles.example/person/example-person",+ proxyUsed: "basic",+ },+ } as any,+ {+ totalCost: 0,+ } as any,+ {} as any,+ undefined,+ undefined,+ { handled: true, creditsCost: 12 },+ );++ expect(credits).toBe(16);+ });+it("bills X/Twitter scrapes at 30 credits", async () => {const credits = await calculateCreditsToBeBilled({diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 0ec7e8733..2de29a4a5 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -153,15 +153,13 @@ export async function calculateCreditsToBeBilled(return creditsToBeBilled + threatScanCredits;}+ // An Exchange access is priced by its provider in place of the base+ // credit; format surcharges (json, question, ...) still apply on top.const exchangeCredits = getExchangeSuccessCredits({exchange,statusCode: document.metadata?.statusCode,});- if (exchangeCredits !== null) {- return exchangeCredits + threatScanCredits;- }-- let creditsToBeBilled = 1; // Assuming 1 credit per document+ let creditsToBeBilled = exchangeCredits ?? 1;if (options.lockdown) {creditsToBeBilled += 4;@@ -202,7 +200,9 @@ export async function calculateCreditsToBeBilled(internalOptions.v1Agent?.model === "fire-1" ||internalOptions.v1JSONAgent?.model?.toLowerCase() === "fire-1") {- creditsToBeBilled = Math.ceil((costTrackingJSON.totalCost ?? 1) * 1800);+ creditsToBeBilled =+ (exchangeCredits ?? 0) ++ Math.ceil((costTrackingJSON.totalCost ?? 1) * 1800);}const hasQuestionFormat =@@ -258,7 +258,10 @@ export async function calculateCreditsToBeBilled(document.metadata?.url,document.metadata?.sourceURL,].filter((u): u is string => !!u);- if (urlsToCheck.some(u => isUrlBlocked(u, null) && !isUrlBlocked(u, flags))) {+ if (+ exchangeCredits === null &&+ urlsToCheck.some(u => isUrlBlocked(u, null) && !isUrlBlocked(u, flags))+ ) {creditsToBeBilled += unblockedDomainCostBonus;}diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex b9f8eb78e..ec3253280 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -29,10 +29,11 @@ import {CREDITS_FEATURE_ID,} from "../services/autumn/autumn.service";import { getTeamBalance } from "../services/autumn/usage";-import { getThirdPartyDataTermsRequiredResponse } from "../lib/exchange";+import { ThirdPartyDataTermsRequiredError } from "../lib/exchange";import { getExchangeAccessForRequestBody } from "../lib/exchange-request";import { isToolsOnlySearch } from "../search/alexandria";import { getScrapeZDR } from "../lib/zdr-helpers";+import { isLockdownZeroDataRetention } from "../lib/safe-mode";import {agentInteropStatus,isAgentInteropSecretValid,@@ -363,11 +364,14 @@ export function blocklistMiddleware(}/**- * Blocklist gate for single-URL scrape-shaped routes (scrape, crawl), where- * an Exchange-eligible URL may bypass the blocklist because the exchange- * engine can serve it. Everything else (map, search, batch scrape, monitors)- * keeps plain blocklist behavior - batch stays out until its jobs carry the- * access flags the worker-side recheck needs.+ * Blocklist gate for single-URL scrape-shaped routes (scrape, crawl), where a+ * blocklisted URL still passes when the exchange engine can serve it, and+ * asks for the provider's terms when unaccepted terms are all that stands in+ * the way. Unblocked URLs never consult the Exchange here: engine selection+ * decides for them, identically on every route, and scrapes them normally+ * when terms are missing. Everything else (map, search, batch scrape,+ * monitors) keeps plain blocklist behavior - batch stays out until its jobs+ * carry the access flags the worker-side recheck needs.*/export function scrapeBlocklistMiddleware(req: RequestWithMaybeACUC<any, any, any>,@@ -384,44 +388,53 @@ function blocklistGate(options: { exchange: boolean },) {(async () => {- const zeroDataRetention =- getScrapeZDR(req.acuc?.flags) === "forced" ||- req.body?.zeroDataRetention === true;- const exchangeAccess =- options.exchange &&- typeof req.body.url === "string" &&- (await getExchangeAccessForRequestBody({+ if (+ typeof req.body.url !== "string" ||+ !isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {+ team_id: req.acuc?.team_id ?? null,+ org_id: req.acuc?.org_id ?? null,+ origin: typeof req.body.origin === "string" ? req.body.origin : null,+ })+ ) {+ return next();+ }++ if (options.exchange) {+ // Safe Mode lockdown implies ZDR, which keeps the Exchange out.+ const zeroDataRetention =+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ req.body?.zeroDataRetention === true ||+ isLockdownZeroDataRetention(req.acuc?.flags, req.body?.safeMode);+ const exchangeAccess = await getExchangeAccessForRequestBody({body: req.body,flags: req.acuc?.flags ?? null,url: req.body.url,+ blocked: true,zeroDataRetention,- }));- const canUseExchange =- typeof exchangeAccess === "object" && exchangeAccess.allowed;+ teamId: req.acuc?.team_id ?? null,+ orgId: req.acuc?.org_id ?? null,+ });- if (typeof exchangeAccess === "object" && exchangeAccess.termsRequired) {- if (!res.headersSent) {+ if (exchangeAccess.allowed) {+ return next();+ }++ if (exchangeAccess.termsRequired && !res.headersSent) {return res.status(403)- .json(getThirdPartyDataTermsRequiredResponse(exchangeAccess.terms));+ .json(+ new ThirdPartyDataTermsRequiredError(+ exchangeAccess.terms,+ ).response(),+ );}}- if (- typeof req.body.url === "string" &&- !canUseExchange &&- isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {- team_id: req.acuc?.team_id ?? null,- org_id: req.acuc?.org_id ?? null,- origin: typeof req.body.origin === "string" ? req.body.origin : null,- })- ) {- if (!res.headersSent) {- return res.status(403).json({- success: false,- error: UNSUPPORTED_SITE_MESSAGE,- });- }+ if (!res.headersSent) {+ return res.status(403).json({+ success: false,+ error: UNSUPPORTED_SITE_MESSAGE,+ });}next();})().catch(err => next(err));diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex 04a758b55..b426fe959 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -17,6 +17,8 @@ type BlockContext = {team_id?: string | null;org_id?: string | null;origin?: string | null;+ /** False for a re-check of a request whose hit was already recorded. */+ record?: boolean;};type BlockHit = {@@ -56,7 +58,7 @@ function recordHit(domain: string,context: BlockContext | undefined,): void {- if (context === undefined) return;+ if (context === undefined || context.record === false) return;if (config.USE_DB_AUTHENTICATION !== true) return;hitBuffer.push({id: uuidv7(),diff --git a/apps/api/src/scraper/scrapeURL/engines/exchange.ts b/apps/api/src/scraper/scrapeURL/engines/exchange.tsindex c6c41de2d..60239dbfc 100644--- a/apps/api/src/scraper/scrapeURL/engines/exchange.ts+++ b/apps/api/src/scraper/scrapeURL/engines/exchange.ts@@ -5,11 +5,14 @@ import { EngineScrapeResult } from ".";import { config } from "../../../config";import {getExchangeRequestLogContext,+ getEnrichmentSettingsUrl,getExchangeResponseLogContext,+ ThirdPartyDataTermsRequiredError,} from "../../../lib/exchange";import { setSpanAttributes, withSpan } from "../../../lib/otel-tracer";import { robustFetch } from "../lib/fetch";-import { EngineError } from "../error";+import { safeMarkdownToHtml } from "./pdf/markdownToHtml";+import { EngineError, ExchangeRefusedError } from "../error";const exchangeScrapeResponseSchema = z.union([z@@ -32,7 +35,6 @@ const exchangeScrapeResponseSchema = z.union([.optional(),metadata: z.record(z.string(), z.unknown()).optional(),markdown: z.string().optional(),- json: z.unknown().optional(),}).passthrough(),})@@ -44,6 +46,7 @@ const exchangeScrapeResponseSchema = z.union([.object({code: z.string().optional(),message: z.string().optional(),+ terms: z.object({ key: z.string(), version: z.string() }).optional(),}).passthrough().optional(),@@ -51,6 +54,50 @@ const exchangeScrapeResponseSchema = z.union([.passthrough(),]);+// Exchange refusals that describe the request itself. Any other failure is an+// engine failure. Enrichment refusals point at the settings that fix them.+const EXCHANGE_REFUSALS = new Map<+ string,+ {+ code: ConstructorParameters<typeof ExchangeRefusedError>[0];+ message: string;+ enrichmentSettings?: true;+ }+>([+ [+ "record_not_found",+ {+ code: "THIRD_PARTY_DATA_NOT_FOUND",+ message:+ "The third-party data provider for this URL has no record for it.",+ },+ ],+ [+ "provider_not_enabled",+ {+ code: "THIRD_PARTY_DATA_NOT_ENABLED",+ message:+ "The third-party data provider for this URL is not enabled for this team.",+ },+ ],+ [+ "enrichment_not_enabled",+ {+ code: "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED",+ message: "Enrichment is not enabled for this kind of profile.",+ enrichmentSettings: true,+ },+ ],+ [+ "enrichment_unavailable",+ {+ code: "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED",+ message: "None of the team's enrichment providers can serve this URL.",+ enrichmentSettings: true,+ },+ ],+]);+export function exchangeMaxReasonableTime(meta: Meta): number {return meta.options.timeout ?? 60_000;}@@ -63,15 +110,21 @@ function escapeHtml(value: string): string {.replace(/"/g, """);}-// Exchange responses carry no page HTML; synthesize a minimal head so the-// metadata transformer can populate the document's title and description.-function buildMetadataHtml(title?: string, description?: string): string {- const titleTag = title === undefined ? "" : `<title>${escapeHtml(title)}</title>`;+// Exchange responses carry no page HTML. Render the markdown into a page so+// the regular transformers derive html, rawHtml, links, images and metadata+// from it the way they do for any other page.+function buildPageHtml(+ body: string,+ title?: string,+ description?: string,+): string {+ const titleTag =+ title === undefined ? "" : `<title>${escapeHtml(title)}</title>`;const descriptionTag =description === undefined? "": `<meta name="description" content="${escapeHtml(description)}">`;- return `<!DOCTYPE html><html><head>${titleTag}${descriptionTag}</head><body></body></html>`;+ return `<!DOCTYPE html><html><head>${titleTag}${descriptionTag}</head><body>${body}</body></html>`;}export async function scrapeURLWithExchange(@@ -105,10 +158,26 @@ export async function scrapeURLWithExchange(requestId: meta.id,teamId: meta.internalOptions.teamId,url,- formats: ["markdown", "json"],+ // Named so the Exchange serves the provider whose access was checked+ // when several claim the URL.+ ...(meta.exchangeProviderId === undefined+ ? {}+ : { provider: meta.exchangeProviderId }),+ formats: ["markdown"],...(meta.options.maxAge === undefined? {}: { maxAge: meta.options.maxAge }),+ // Enrichment checks each provider step's terms against these rows+ // and the Exchange's own ledger for the organization.+ ...(meta.internalOptions.orgId+ ? { organizationId: meta.internalOptions.orgId }+ : {}),+ ...(meta.internalOptions.teamFlags?.organizationDataSourceAccess+ ? {+ organizationDataSourceAccess:+ meta.internalOptions.teamFlags.organizationDataSourceAccess,+ }+ : {}),},logger: logger.child({ method: "exchangeScrape/robustFetch" }),tryCount: 2,@@ -126,6 +195,26 @@ export async function scrapeURLWithExchange(errorCode: response.error?.code,durationMs: Date.now() - startTime,});+ if (+ response.error?.code === "third_party_data_terms_required" &&+ response.error.terms !== undefined+ ) {+ // Only enrichment checks terms inside the Exchange, so the step can+ // also be turned off in the team's enrichment settings.+ throw new ThirdPartyDataTermsRequiredError(response.error.terms, {+ enrichment: true,+ });+ }+ const refusal = EXCHANGE_REFUSALS.get(response.error?.code ?? "");+ if (refusal !== undefined) {+ const message = response.error?.message || refusal.message;+ throw new ExchangeRefusedError(+ refusal.code,+ refusal.enrichmentSettings+ ? `${message} An organization admin can choose enrichment providers at ${getEnrichmentSettingsUrl()}`+ : message,+ );+ }throw new EngineError("Exchange request failed");}@@ -152,11 +241,15 @@ export async function scrapeURLWithExchange("exchange.duration_ms": Date.now() - startTime,});+ const markdown = response.data.markdown ?? "";return {url: response.data.url ?? url,- html: buildMetadataHtml(response.data.title, response.data.description),- markdown: response.data.markdown,- json: response.data.json,+ html: buildPageHtml(+ await safeMarkdownToHtml(markdown, meta.logger, meta.id),+ response.data.title,+ response.data.description,+ ),+ markdown,statusCode: 200,contentType: "text/markdown",proxyUsed: "basic",diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex e96279522..b9d7c4db2 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -34,7 +34,6 @@ const successSchema = z.object({// timeTaken: z.number(),content: z.string(),- json: z.unknown().optional(),url: z.string().optional(),pageStatusCode: z.number(),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 4c533e830..33a60027f 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -597,7 +597,6 @@ export async function scrapeURLWithFireEngineChromeCDP(markdown: contentType?.includes("text/markdown")? response.content: undefined,- json: response.json,error: response.pageError,statusCode: response.pageStatusCode,@@ -698,7 +697,6 @@ export async function scrapeURLWithFireEngineTLSClient(markdown: contentType?.includes("text/markdown")? response.content: undefined,- json: response.json,error: response.pageError,statusCode: response.pageStatusCode,diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex d628a23c5..1f5b69927 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -112,7 +112,6 @@ const successSchema = z.object({timeTaken: z.number(),content: z.string(),- json: z.unknown().optional(),url: z.string().optional(),pageStatusCode: z.number(),diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex d2207668b..fb38ebf31 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -44,7 +44,8 @@ import {import { isUrlBlocked } from "../../WebScraper/utils/blocklist";import { hasCustomRequestContext } from "../lib/request-context";import {- canUseExchangeForRequest,+ getExchangeAccessForRequest,+ ThirdPartyDataTermsRequiredError,type ExchangeScrapeMetadata,} from "../../../lib/exchange";@@ -156,7 +157,6 @@ export type EngineScrapeResult = {markdown?: string;pages?: Array<{ pageNumber: number; markdown: string }>;blocks?: PdfPageBlocks[];- json?: unknown;statusCode: number;error?: string;@@ -665,27 +665,54 @@ export async function buildFallbackList(meta: Meta): Promise<!meta.internalOptions.agentIndexOnly &&meta.internalOptions.forceEngine === undefined) {- if (- await canUseExchangeForRequest({- url: meta.rewrittenUrl ?? meta.url,- formats: meta.options.formats,- actions: meta.options.actions,- headers: meta.options.headers,- waitFor: meta.options.waitFor,- mobile: meta.options.mobile,- location: meta.options.location,- proxy: meta.options.proxy,- blockAds: meta.options.blockAds,- profile: meta.options.profile,- atsv: meta.internalOptions.atsv,- minAge: meta.options.minAge,- includeTags: meta.options.includeTags,- excludeTags: meta.options.excludeTags,- zeroDataRetention: meta.internalOptions.zeroDataRetention,- lockdown: meta.options.lockdown,- flags: meta.internalOptions.teamFlags ?? null,- })- ) {+ const url = meta.rewrittenUrl ?? meta.url;+ const betaTeam =+ meta.internalOptions.teamFlags?.professionalProfileCompanyDataBeta ===+ true;+ let blocked = false;+ try {+ blocked = isUrlBlocked(url, meta.internalOptions.teamFlags ?? null, {+ team_id: meta.internalOptions.teamId ?? null,+ org_id: meta.internalOptions.orgId ?? null,+ origin: null,+ record: false,+ });+ } catch (error) {+ // Beta teams fail closed, as they always have. Anyone else keeps the+ // normal engines, since an unreadable blocklist must not fail every+ // scrape.+ if (betaTeam) {+ meta.logger.warn("Exchange blocklist check failed; failing closed", {+ error,+ });+ return [];+ }+ }++ const exchangeAccess = await getExchangeAccessForRequest({+ url,+ teamId: meta.internalOptions.teamId ?? null,+ orgId: meta.internalOptions.orgId ?? null,+ blocked,+ formats: meta.options.formats,+ actions: meta.options.actions,+ headers: meta.options.headers,+ waitFor: meta.options.waitFor,+ mobile: meta.options.mobile,+ location: meta.options.location,+ proxy: meta.options.proxy,+ blockAds: meta.options.blockAds,+ profile: meta.options.profile,+ atsv: meta.internalOptions.atsv,+ minAge: meta.options.minAge,+ includeTags: meta.options.includeTags,+ excludeTags: meta.options.excludeTags,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,+ lockdown: meta.options.lockdown,+ flags: meta.internalOptions.teamFlags ?? null,+ });+ if (exchangeAccess.allowed) {+ meta.exchangeProviderId = exchangeAccess.provider.id;return [{engine: "exchange",@@ -694,37 +721,16 @@ export async function buildFallbackList(meta: Meta): Promise<];}- // A blocked URL can only have been admitted by the Exchange bypass in- // blocklistMiddleware, which only applies to flagged orgs; if the- // Exchange is no longer usable by execution time (catalog changed,- // service down), fail closed rather than letting normal engines scrape- // a blocklisted site. An error here also fails closed: this branch only- // runs for flagged orgs, and a retryable scrape failure is preferable- // to scraping a potentially blocklisted site with normal engines.- if (- meta.internalOptions.teamFlags?.professionalProfileCompanyDataBeta ===- true- ) {- try {- if (- isUrlBlocked(- meta.rewrittenUrl ?? meta.url,- meta.internalOptions.teamFlags ?? null,- {- team_id: meta.internalOptions.teamId ?? null,- org_id: meta.internalOptions.orgId ?? null,- origin: null,- },- )- ) {- return [];- }- } catch (error) {- meta.logger.warn("Exchange blocklist re-check failed; failing closed", {- error,- });- return [];+ // A blocked URL can only have been admitted by scrapeBlocklistMiddleware+ // for the Exchange; if the Exchange can no longer serve it (catalog+ // changed, service down), fail closed rather than letting normal engines+ // scrape a blocklisted site. Unblocked URLs whose provider wants+ // unaccepted terms fall through and scrape normally.+ if (blocked) {+ if (exchangeAccess.termsRequired) {+ throw new ThirdPartyDataTermsRequiredError(exchangeAccess.terms);}+ return [];}}diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex d2e652bb1..15c065291 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -120,7 +120,6 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {meta.rewrittenUrl ??meta.url,html: document.rawHtml!,- json: document.json,statusCode: document.metadata.statusCode,error: document.metadata.error,screenshot: document.screenshot,@@ -588,7 +587,6 @@ export async function scrapeURLWithIndex(return {url: doc.url,html: doc.html,- json: doc.json,statusCode: doc.statusCode,error: doc.error,screenshot: doc.screenshot,diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 19129fdb7..59bc1beec 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -121,6 +121,32 @@ export class SSLError extends TransportableError {}}+type ExchangeRefusalCode =+ | "THIRD_PARTY_DATA_NOT_FOUND"+ | "THIRD_PARTY_DATA_NOT_ENABLED"+ | "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED";++// A definitive answer from the Exchange about this URL: the provider holds no+// record for it, or the team is not entitled to the provider. Another attempt+// cannot change it, so it surfaces as-is instead of as an engine failure.+export class ExchangeRefusedError extends TransportableError {+ constructor(code: ExchangeRefusalCode, message: string) {+ super(code, message);+ }++ static deserialize(+ code: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new ExchangeRefusedError(+ code as ExchangeRefusalCode,+ data.message,+ );+ x.stack = data.stack;+ return x;+ }+}+export class SiteError extends TransportableError {constructor(public errorCode: string) {const errorExplanations: Record<string, string> = {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 36c264b3b..16f6ae239 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -60,6 +60,7 @@ import {ScrapeRetryLimitError,BrandingNotSupportedError,XTwitterConfigurationError,+ ExchangeRefusedError,} from "./error";import { ScrapeRetryTracker } from "./retryTracker";import { executeTransformers } from "./transformers";@@ -106,7 +107,10 @@ import { writeFile } from "node:fs/promises";import path from "node:path";import { tmpdir } from "node:os";import { randomUUID } from "node:crypto";-import type { ExchangeScrapeMetadata } from "../../lib/exchange";+import {+ type ExchangeScrapeMetadata,+ ThirdPartyDataTermsRequiredError,+} from "../../lib/exchange";import {checkUrl,type ThreatCheckDedup,@@ -167,6 +171,9 @@ export type Meta = {abort: AbortManager;featureFlags: Set<FeatureFlag>;mock: MockState | null;+ /** The Exchange provider whose access was checked when the exchange engine+ * was chosen, so the engine asks the Exchange for that same provider. */+ exchangeProviderId?: string;/** Whether this scrape may OCR raster images: the request's parsers* include `image` (the default; a parse upload of an image always counts)* and the deployment has image OCR switched on with FirePDF configured.@@ -1006,7 +1013,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error.error instanceof ProxySelectionError ||error.error instanceof NoCachedDataError ||error.error instanceof AgentIndexOnlyError ||- error.error instanceof XTwitterConfigurationError+ error.error instanceof XTwitterConfigurationError ||+ error.error instanceof ExchangeRefusedError ||+ error.error instanceof ThirdPartyDataTermsRequiredError) {throw error.error;} else if (error.error instanceof LLMRefusalError) {@@ -1155,7 +1164,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {blocks: engineResult.blocks,rawHtml: engineResult.html,rawBase64: engineResult.rawBase64,- json: engineResult.json,screenshot: engineResult.screenshot,actions: engineResult.actions,branding: engineResult.branding,diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.test.ts b/apps/api/src/scraper/scrapeURL/transformers/index.test.tsindex 1e87844b0..989c451e1 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.test.ts@@ -37,15 +37,20 @@ describe("executeTransformers", () => {mockedPerformLLMExtract.mockClear();});- it("keeps native JSON and markdown without running LLM JSON extraction", async () => {- const nativeJson = { id: "person-1", full_name: "Example Person" };- const nativeMarkdown = "# Example Person";+ it("runs JSON extraction on markdown an engine delivered natively", async () => {+ mockedPerformLLMExtract.mockImplementationOnce(async (_meta, document) => ({+ ...document,+ json: { name: "Example Person" },+ }));const document = await executeTransformers({url: "https://www.linkedin.com/in/example",options: {- formats: [{ type: "markdown" }, { type: "json" }],+ formats: [+ { type: "markdown" },+ { type: "json", schema: { type: "object" } },+ ],onlyMainContent: false,},internalOptions: {},@@ -53,41 +58,8 @@ describe("executeTransformers", () => {} as any,{rawHtml:- "<html><head><title>Ignored</title></head><body></body></html>",- markdown: nativeMarkdown,- json: nativeJson,- metadata: {- sourceURL: "https://www.linkedin.com/in/example",- url: "https://www.linkedin.com/in/example",- statusCode: 200,- contentType: "text/markdown; charset=utf-8",- },- } as any,- );-- expect(mockedPerformLLMExtract).not.toHaveBeenCalled();- expect(document.markdown).toBe(nativeMarkdown);- expect(document.json).toEqual(nativeJson);- });-- it("maps native JSON to v1 extract compatibility field", async () => {- const nativeJson = { id: "person-1", full_name: "Example Person" };-- const document = await executeTransformers(- {- url: "https://www.linkedin.com/in/example",- options: {- formats: [{ type: "markdown" }, { type: "json" }],- onlyMainContent: false,- },- internalOptions: { v1OriginalFormat: "extract" },- logger: logger(),- } as any,- {- rawHtml:- "<html><head><title>Ignored</title></head><body></body></html>",+ "<html><head><title>Example</title></head><body><h1>Example Person</h1></body></html>",markdown: "# Example Person",- json: nativeJson,metadata: {sourceURL: "https://www.linkedin.com/in/example",url: "https://www.linkedin.com/in/example",@@ -97,8 +69,8 @@ describe("executeTransformers", () => {} as any,);- expect(mockedPerformLLMExtract).not.toHaveBeenCalled();- expect(document.extract).toEqual(nativeJson);- expect(document.json).toEqual(nativeJson);+ expect(mockedPerformLLMExtract).toHaveBeenCalledTimes(1);+ expect(document.markdown).toBe("# Example Person");+ expect(document.json).toEqual({ name: "Example Person" });});});diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex e447aab2d..c899263c4 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -339,30 +339,6 @@ async function deriveBrandingFromActions(return document;}-async function performLLMExtractUnlessNativeJson(- meta: Meta,- document: Document,-): Promise<Document> {- if (- document.json !== undefined &&- hasFormatOfType(meta.options.formats, "json")- ) {- if (- meta.internalOptions.v1OriginalFormat === "extract" &&- document.extract === undefined- ) {- document.extract = document.json;- }-- meta.logger.debug(- "Skipping LLM JSON extraction - document already has native JSON",- );- return document;- }-- return performLLMExtract(meta, document);-}-function coerceFieldsToFormats(meta: Meta, document: Document): Document {const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");const hasRawHtml = hasFormatOfType(meta.options.formats, "rawHtml");@@ -653,7 +629,7 @@ const transformerStack: Transformer[] = [fetchMenu,...(useIndex ? [sendDocumentToIndex] : []),...(useSearchIndex ? [sendDocumentToSearchIndex] : []), // Add to search index for real-time search- performLLMExtractUnlessNativeJson,+ performLLMExtract,performDeterministicJson,performSummary,performQuery,diff --git a/apps/api/src/services/alexandria/access.ts b/apps/api/src/services/alexandria/access.tsindex c4f8c50b4..ec317d82a 100644--- a/apps/api/src/services/alexandria/access.ts+++ b/apps/api/src/services/alexandria/access.ts@@ -1,10 +1,15 @@import { z } from "zod";import { config } from "../../config";import type { TeamFlags } from "../../controllers/v2/types";-import { getThirdPartyDataTermsRequiredResponse } from "../../lib/exchange";+import { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";import { exchangeRequest } from "./client";import { refusal, type ExchangeResponse, type ProviderCall } from "./contracts";-import { acceptedProviders, type LedgerAcceptance } from "./terms";+import {+ acceptedAfter,+ acceptedProviders,+ matchesAcceptance,+ type LedgerAcceptance,+} from "./terms";import { autumnService } from "../autumn/autumn.service";import { HOBBY_RATE_LIMIT_MULTIPLIER } from "../rate-limiter";@@ -29,24 +34,6 @@ const requirementsSchema = z.object({),});-const timestampSchema = z.iso.datetime({ offset: true });--function timestamp(value: unknown): number {- const parsed = timestampSchema.safeParse(value);- return parsed.success ? Date.parse(parsed.data) : NaN;-}--function matchesAcceptance(- accepted: LedgerAcceptance | undefined,- terms: { version: string; digest?: string },-): boolean {- return (- terms.digest !== undefined &&- accepted?.version === terms.version &&- accepted.textHash === terms.digest- );-}-export async function authorizeProviders(teamId: string,calls: ProviderCall[],@@ -92,18 +79,14 @@ export async function authorizeProviders(if (revokedByOwner && item.required && item.terms && orgId !== null) {ledger ??= await acceptedProviders(teamId, orgId);const accepted = ledger.get(item.provider);- const acceptedAt = timestamp(accepted?.acceptedAt);- const disabledAt = timestamp(access.disabledAt);if (matchesAcceptance(accepted, item.terms) &&- Number.isFinite(acceptedAt) &&- Number.isFinite(disabledAt) &&- acceptedAt > disabledAt+ acceptedAfter(accepted, access.disabledAt))continue;return {status: 403,- body: getThirdPartyDataTermsRequiredResponse(item.terms),+ body: new ThirdPartyDataTermsRequiredError(item.terms).response(),};}return refusal(@@ -124,7 +107,7 @@ export async function authorizeProviders(}return {status: 403,- body: getThirdPartyDataTermsRequiredResponse(item.terms),+ body: new ThirdPartyDataTermsRequiredError(item.terms).response(),};}diff --git a/apps/api/src/services/alexandria/terms.ts b/apps/api/src/services/alexandria/terms.tsindex deff0a7e2..8f042bb7c 100644--- a/apps/api/src/services/alexandria/terms.ts+++ b/apps/api/src/services/alexandria/terms.ts@@ -57,14 +57,48 @@ export type LedgerAcceptance = {acceptedAt?: string | null;};+const timestampSchema = z.iso.datetime({ offset: true });++function timestamp(value: unknown): number {+ const parsed = timestampSchema.safeParse(value);+ return parsed.success ? Date.parse(parsed.data) : NaN;+}++export function matchesAcceptance(+ accepted: LedgerAcceptance | undefined,+ terms: { version: string; digest?: string },+): boolean {+ return (+ terms.digest !== undefined &&+ accepted?.version === terms.version &&+ accepted.textHash === terms.digest+ );+}++// Whether the acceptance was recorded after `since`, e.g. an admin revocation+// it has to lift. Unparseable timestamps never qualify.+export function acceptedAfter(+ accepted: LedgerAcceptance | undefined,+ since: unknown,+): boolean {+ const acceptedAt = timestamp(accepted?.acceptedAt);+ const sinceAt = timestamp(since);+ return (+ Number.isFinite(acceptedAt) &&+ Number.isFinite(sinceAt) &&+ acceptedAt > sinceAt+ );+}+export async function acceptedProviders(teamId: string,orgId: string,+ timeoutMs = TIMEOUT_MS,): Promise<Map<string, LedgerAcceptance>> {const response = await exchangeRequest({teamId,path: `/v1/provider-terms/status?organizationId=${encodeURIComponent(orgId)}`,- timeoutMs: TIMEOUT_MS,+ timeoutMs,}).catch(() => undefined);const parsed =response?.status === 200@@ -84,6 +118,45 @@ export async function acceptedProviders(return accepted;}+/**+ * Whether the organization's ledger on the Exchange holds an acceptance of the+ * provider's current terms, recorded after `revocation.disabledAt` when an+ * admin revocation has to be lifted. The API's accept route records+ * acceptance only there. Any failure reads as not accepted.+ */+export async function hasLedgerAcceptance(input: {+ teamId: string;+ orgId: string;+ provider: string;+ revocation?: { disabledAt: unknown };+ timeoutMs: number;+}): Promise<boolean> {+ const [requirements, ledger] = await Promise.all([+ exchangeRequest({+ teamId: input.teamId,+ path: "/v1/provider-terms/requirements",+ body: { providers: [input.provider] },+ timeoutMs: input.timeoutMs,+ }).catch(() => undefined),+ acceptedProviders(input.teamId, input.orgId, input.timeoutMs),+ ]);+ const parsed =+ requirements?.status === 200+ ? requirementsSchema.safeParse(requirements.body)+ : undefined;+ const terms = parsed?.success+ ? parsed.data.providers.find(item => item.provider === input.provider)+ ?.terms+ : undefined;+ if (!terms) return false;+ const accepted = ledger.get(input.provider);+ return (+ matchesAcceptance(accepted, terms) &&+ (input.revocation === undefined ||+ acceptedAfter(accepted, input.revocation.disabledAt))+ );+}+export async function acceptProviderTerms(input: {teamId: string;orgId: string;diff --git a/apps/api/src/services/index.test.ts b/apps/api/src/services/index.test.tsdeleted file mode 100644index ba7507e7d..000000000--- a/apps/api/src/services/index.test.ts+++ /dev/null@@ -1,105 +0,0 @@-import { vi } from "vitest";--const { file, bucket, getSavedPayload, setSavedPayload } = vi.hoisted(() => {- let savedPayload = "";- const getSavedPayload = () => savedPayload;- const setSavedPayload = (value: string) => {- savedPayload = value;- };- const file = vi.fn(() => ({- save: vi.fn(async (payload: string) => {- savedPayload = payload;- }),- download: vi.fn(async () => [Buffer.from(savedPayload)]),- }));- const bucket = vi.fn(() => ({ file }));-- return { file, bucket, getSavedPayload, setSavedPayload };-});--vi.mock("../config", () => ({- config: {- GCS_INDEX_BUCKET_NAME: "index-bucket",- GCS_MEDIA_BUCKET_NAME: "media-bucket",- },-}));--vi.mock("../lib/gcs-jobs", () => ({- storage: { bucket },-}));--vi.mock("../lib/otel-tracer", () => ({- withSpan: vi.fn(async (_name, fn) => fn({})),- setSpanAttributes: vi.fn(),-}));--vi.mock("../db/connection", () => ({- dbIndex: {},-}));--vi.mock("../db/rpc", () => ({- insertOmceJobIfNeeded: vi.fn(),- queryIndexAtSplitLevel: vi.fn(),- queryIndexAtDomainSplitLevel: vi.fn(),- queryOmceSignatures: vi.fn(),- queryEngpickerVerdict: vi.fn(),- queryIndexAtSplitLevelWithMeta: vi.fn(),- queryIndexAtDomainSplitLevelWithMeta: vi.fn(),- queryDomainPriority: vi.fn(),-}));--vi.mock("./redis", () => ({- redisEvictConnection: {},-}));--vi.mock("./index-cache", () => ({- deriveIndexVariantKey: vi.fn(),- upsertCachedIndexEntries: vi.fn(),- useIndexCache: false,-}));--vi.mock("../db/schema", () => ({}));--import { getIndexFromGCS, saveIndexToGCS } from "./index";--describe("index GCS documents", () => {- beforeEach(() => {- vi.clearAllMocks();- setSavedPayload("");- });-- it("persists native JSON payloads with cached index documents", async () => {- await saveIndexToGCS("idx-1", {- url: "https://www.example.com/profile",- html: "# Example",- json: { id: "person-1", full_name: "Example Person" },- statusCode: 200,- contentType: "text/markdown; charset=utf-8",- proxyUsed: "basic",- });-- expect(file).toHaveBeenCalledWith("idx-1.json");- expect(JSON.parse(getSavedPayload()).json).toEqual({- id: "person-1",- full_name: "Example Person",- });- });-- it("reads native JSON payloads from cached index documents", async () => {- setSavedPayload(- JSON.stringify({- url: "https://www.example.com/profile",- html: "# Example",- json: { id: "person-1", full_name: "Example Person" },- statusCode: 200,- }),- );-- const doc = await getIndexFromGCS("idx-1.json");-- expect(doc?.json).toEqual({- id: "person-1",- full_name: "Example Person",- });- });-});diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex ceef42295..3a047e6ee 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -118,7 +118,6 @@ export async function saveIndexToGCS(doc: {url: string;html: string;- json?: unknown;statusCode: number;error?: string;screenshot?: string;diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex 9f4bbc47c..8b0564bf1 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -2,6 +2,7 @@ import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } fromimport * as zt from "zod";import { zodSchemaToJsonSchema } from "../utils/zodSchemaToJson";import { TypedEventTarget } from "typescript-event-target";+import type { RequiresAction } from "../v2/types";/*** Configuration interface for FirecrawlApp.@@ -439,6 +440,8 @@ export interface CrawlErrorsResponse {url: string,code?: string,error: string,+ /** Present when the page needs provider terms accepted first. */+ requiresAction?: RequiresAction,}[];/**diff --git a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts b/apps/js-sdk/firecrawl/src/v2/methods/batch.tsindex 90bfd4665..9683cffae 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts@@ -152,7 +152,7 @@ export async function getBatchScrapeErrors(try {const res = await http.get<{success?: boolean;- data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] };+ data?: CrawlErrorsResponse;}>(`/v2/batch/scrape/${jobId}/errors`);if (res.status !== 200) throwForBadResponse(res, "get batch scrape errors");const payload = res.data?.data ?? (res.data as any);diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.tsindex 7a46c5aad..834444c13 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts@@ -162,7 +162,7 @@ export async function crawl(http: HttpClient, request: CrawlRequest, pollIntervaexport async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {try {- const res = await http.get<{ success?: boolean; data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] } }>(`/v2/crawl/${crawlId}/errors`);+ const res = await http.get<{ success?: boolean; data?: CrawlErrorsResponse }>(`/v2/crawl/${crawlId}/errors`);if (res.status !== 200) throwForBadResponse(res, "get crawl errors");const payload = res.data?.data ?? (res.data as any);return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex b048701ed..fb0b6f8fc 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -1955,6 +1955,8 @@ export interface CrawlErrorsResponse {url: string;code?: string;error: string;+ /** Present when the page needs provider terms accepted first. */+ requiresAction?: RequiresAction;}[];robotsBlocked: string[];}diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_errors.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_errors.pynew file mode 100644index 000000000..4951d4042--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_errors.py@@ -0,0 +1,61 @@+from unittest.mock import Mock++from firecrawl.v1.client import V1CrawlErrorsResponse+from firecrawl.v2.methods.batch import get_batch_scrape_errors+from firecrawl.v2.methods.crawl import get_crawl_errors++REQUIRES_ACTION = {+ "type": "accept_terms",+ "terms": "acme",+ "version": "2026-01-01",+ "url": "https://www.firecrawl.dev/app/alexandria/acme",+}++ERRORS = {+ "errors": [+ {+ "id": "job-terms",+ "url": "https://profiles.example/person/example-person",+ "code": "THIRD_PARTY_DATA_TERMS_REQUIRED",+ "error": "An organization admin must accept the acme provider's terms.",+ "requiresAction": REQUIRES_ACTION,+ },+ {+ "id": "job-site",+ "url": "https://down.example/",+ "code": "SCRAPE_SITE_ERROR",+ "error": "The connection was reset by the peer.",+ },+ ],+ "robotsBlocked": [],+}+++def _client():+ response = Mock()+ response.ok = True+ response.json.return_value = ERRORS+ client = Mock()+ client.get.return_value = response+ return client+++def test_crawl_errors_carry_requires_action():+ result = get_crawl_errors(_client(), "crawl-1")++ assert result.errors[0].requires_action == REQUIRES_ACTION+ assert result.errors[1].requires_action is None+++def test_batch_scrape_errors_carry_requires_action():+ result = get_batch_scrape_errors(_client(), "batch-1")++ assert result.errors[0].requires_action == REQUIRES_ACTION+ assert result.errors[1].requires_action is None+++def test_v1_crawl_errors_carry_requires_action():+ result = V1CrawlErrorsResponse(**ERRORS)++ assert result.errors[0].requiresAction == REQUIRES_ACTION+ assert result.errors[1].requiresAction is Nonediff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.pyindex c3c1b884b..b20a32096 100644--- a/apps/python-sdk/firecrawl/v1/client.py+++ b/apps/python-sdk/firecrawl/v1/client.py@@ -295,6 +295,9 @@ class V1CrawlError(pydantic.BaseModel):url: strcode: Optional[str] = Noneerror: str+ # Set when the page needs provider terms accepted first:+ # {"type": "accept_terms", "terms", "version", "url"}.+ requiresAction: Optional[Dict[str, Any]] = Noneclass V1CrawlErrorsResponse(pydantic.BaseModel):"""Response from crawl/batch scrape error monitoring."""diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex fda30a374..ecf2f4dab 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -2544,11 +2544,18 @@ class JobStatus(BaseModel):class CrawlError(BaseModel):"""A crawl error."""+ model_config = {"populate_by_name": True}+id: strtimestamp: Optional[datetime] = Noneurl: strcode: Optional[str] = Noneerror: str+ # Set when the page needs provider terms accepted first:+ # {"type": "accept_terms", "terms", "version", "url"}.+ requires_action: Optional[Dict[str, Any]] = Field(+ default=None, alias="requiresAction"+ )class CrawlErrorsResponse(BaseModel):