Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): route-independent Exchange terms handling and LinkedIn enrichment routing (#4846)

Gergő Móricz33c042fd1e3c
diff --git a/apps/api/src/__tests__/snips/v2/crawl-errors-terms.test.ts b/apps/api/src/__tests__/snips/v2/crawl-errors-terms.test.ts
new file mode 100644
index 000000000..48e050bf5
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/crawl-errors-terms.test.ts
@@ -0,0 +1,88 @@
+/**
+ * Unit test for the crawl/batch errors listing: a page that failed on
+ * unaccepted provider terms carries the same `requiresAction` a single
+ * scrape's 403 body does, on v1 and v2.
+ */
+
+const mocks = vi.hoisted(() => ({
+ getCrawl: vi.fn(),
+ getGroupJobs: vi.fn(),
+ smembers: vi.fn(),
+}));
+
+vi.mock("../../../lib/crawl-redis", () => ({
+ getCrawl: mocks.getCrawl,
+}));
+vi.mock("../../../services/worker/nuq-router", () => ({
+ scrapeQueue: { getGroupJobs: mocks.getGroupJobs },
+}));
+vi.mock("../../../services/redis", () => ({
+ redisEvictConnection: { smembers: mocks.smembers },
+}));
+
+import { crawlErrorsController as v1CrawlErrorsController } from "../../../controllers/v1/crawl-errors";
+import { crawlErrorsController as v2CrawlErrorsController } from "../../../controllers/v2/crawl-errors";
+import { ThirdPartyDataTermsRequiredError } from "../../../lib/exchange";
+import { serializeTransportableError } from "../../../lib/error-serde";
+import { SiteError } from "../../../scraper/scrapeURL/error";
+
+describe.each([
+ ["v1", v1CrawlErrorsController],
+ ["v2", v2CrawlErrorsController],
+])("%s crawl errors", (_version, crawlErrorsController) => {
+ const TERMS = { key: "acme", version: "2026-01-01" };
+
+ beforeEach(() => {
+ mocks.getCrawl.mockResolvedValue({ team_id: "team-1" });
+ mocks.smembers.mockResolvedValue([]);
+ mocks.getGroupJobs.mockResolvedValue([
+ {
+ id: "job-terms",
+ data: { url: "https://profiles.example/person/example-person" },
+ failedReason: serializeTransportableError(
+ new ThirdPartyDataTermsRequiredError(TERMS),
+ ),
+ },
+ {
+ id: "job-site",
+ data: { url: "https://down.example/" },
+ failedReason: serializeTransportableError(
+ new SiteError("ERR_CONNECTION_RESET"),
+ ),
+ },
+ ]);
+ });
+
+ async function listErrors() {
+ let body: any;
+ const res: any = {
+ status: () => res,
+ json: (payload: unknown) => {
+ body = payload;
+ return res;
+ },
+ };
+ await crawlErrorsController(
+ {
+ params: { jobId: "crawl-1" },
+ auth: { team_id: "team-1" },
+ } as any,
+ res,
+ );
+ return body;
+ }
+
+ it("carries requiresAction on terms-required entries only", async () => {
+ const { errors } = await listErrors();
+ const expected = new ThirdPartyDataTermsRequiredError(TERMS).response();
+
+ expect(errors.find((e: any) => e.id === "job-terms")).toMatchObject({
+ code: "THIRD_PARTY_DATA_TERMS_REQUIRED",
+ error: expected.error,
+ requiresAction: expected.requiresAction,
+ });
+ expect(errors.find((e: any) => e.id === "job-site")).not.toHaveProperty(
+ "requiresAction",
+ );
+ });
+});
diff --git a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts
new file mode 100644
index 000000000..531c99840
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts
@@ -0,0 +1,295 @@
+/**
+ * Unit tests for Exchange routing: the scrape blocklist gate, engine
+ * selection in buildFallbackList, and how the exchange engine reports the
+ * Exchange's refusals. Unaccepted terms only surface as an error when they
+ * are all that keeps the Exchange from serving a blocklisted URL; any other
+ * URL scrapes normally, the same way on every route.
+ */
+
+const mocks = vi.hoisted(() => ({
+ isUrlBlocked: vi.fn(
+ (url: string) => new URL(url).hostname === "profiles.example",
+ ),
+ robustFetch: vi.fn(),
+ exchangeRequest: vi.fn(),
+}));
+
+vi.mock("../../../scraper/WebScraper/utils/blocklist", () => ({
+ isUrlBlocked: mocks.isUrlBlocked,
+}));
+vi.mock("../../../scraper/scrapeURL/lib/fetch", () => ({
+ robustFetch: mocks.robustFetch,
+}));
+vi.mock("../../../services/alexandria/client", () => ({
+ exchangeRequest: mocks.exchangeRequest,
+}));
+
+describe("Exchange routing", () => {
+ let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;
+ let scrapeURLWithExchange: typeof import("../../../scraper/scrapeURL/engines/exchange.js").scrapeURLWithExchange;
+ let scrapeBlocklistMiddleware: typeof import("../../../routes/shared.js").scrapeBlocklistMiddleware;
+ let exchange: typeof import("../../../lib/exchange.js");
+ let scrapeErrors: typeof import("../../../scraper/scrapeURL/error.js");
+ let config: typeof import("../../../config.js").config;
+ let originalUseDbAuthentication: boolean | undefined;
+
+ const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;
+ const originalExchangeUrl = process.env.FIRE_EXCHANGE_URL;
+
+ const TERMS = { key: "acme", version: "2026-01-01" };
+ const BLOCKED_URL = "https://profiles.example/person/example-person";
+ const OPEN_URL = "https://open.example/person/example-person";
+
+ const ACCEPTED_FLAGS = {
+ professionalProfileCompanyDataBeta: true,
+ organizationDataSourceAccess: {
+ acme: {
+ status: "enabled",
+ termsKey: TERMS.key,
+ termsVersion: TERMS.version,
+ },
+ },
+ };
+ const UNACCEPTED_FLAGS = { professionalProfileCompanyDataBeta: true };
+
+ beforeAll(async () => {
+ process.env.FIRE_ENGINE_BETA_URL = "http://test-fire-engine";
+ process.env.FIRE_EXCHANGE_URL = "http://test-exchange";
+
+ // Re-import so modules read the env vars set above at eval time.
+ vi.resetModules();
+ ({ buildFallbackList } = await import(
+ "../../../scraper/scrapeURL/engines/index.js"
+ ));
+ ({ scrapeURLWithExchange } = await import(
+ "../../../scraper/scrapeURL/engines/exchange.js"
+ ));
+ ({ scrapeBlocklistMiddleware } = await import("../../../routes/shared.js"));
+ exchange = await import("../../../lib/exchange.js");
+ scrapeErrors = await import("../../../scraper/scrapeURL/error.js");
+ ({ config } = await import("../../../config.js"));
+ });
+
+ beforeEach(() => {
+ originalUseDbAuthentication = config.USE_DB_AUTHENTICATION;
+ config.USE_DB_AUTHENTICATION = true;
+ // The ledger is unreachable: only the flags can show acceptance.
+ mocks.exchangeRequest.mockRejectedValue(new Error("offline"));
+ mocks.robustFetch.mockReset();
+ exchange.setExchangeProvidersForTest([
+ {
+ id: "acme",
+ creditsCost: 12,
+ terms: TERMS,
+ routes: [{ domains: ["profiles.example", "open.example"] }],
+ },
+ ]);
+ });
+
+ afterEach(() => {
+ config.USE_DB_AUTHENTICATION = originalUseDbAuthentication;
+ exchange.clearExchangeProvidersForTest();
+ });
+
+ afterAll(() => {
+ if (originalFireEngineUrl === undefined) {
+ delete process.env.FIRE_ENGINE_BETA_URL;
+ } else {
+ process.env.FIRE_ENGINE_BETA_URL = originalFireEngineUrl;
+ }
+ if (originalExchangeUrl === undefined) {
+ delete process.env.FIRE_EXCHANGE_URL;
+ } else {
+ process.env.FIRE_EXCHANGE_URL = originalExchangeUrl;
+ }
+ });
+
+ const buildStubMeta = (url: string, teamFlags: Record<string, unknown>) =>
+ ({
+ id: "test",
+ url,
+ options: {
+ formats: [{ type: "markdown" }],
+ maxAge: 3600000,
+ },
+ internalOptions: { teamId: "team-1", orgId: "org-1", teamFlags },
+ featureFlags: new Set(),
+ mock: null,
+ abort: { asSignal: () => undefined },
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ debug: vi.fn(),
+ error: vi.fn(),
+ child: vi.fn().mockReturnThis(),
+ },
+ }) as any;
+
+ // Runs the scrape blocklist gate and resolves with the response it sent,
+ // or `next` when it let the request through.
+ function runGate(
+ body: Record<string, unknown>,
+ flags: Record<string, unknown>,
+ ): Promise<{ next: true } | { status: number; body: any }> {
+ return new Promise((resolve, reject) => {
+ let statusCode = 200;
+ const res: any = {
+ headersSent: false,
+ status(code: number) {
+ statusCode = code;
+ return res;
+ },
+ json(payload: unknown) {
+ resolve({ status: statusCode, body: payload });
+ return res;
+ },
+ };
+ scrapeBlocklistMiddleware(
+ { body, acuc: { flags, team_id: "team-1", org_id: "org-1" } } as any,
+ res,
+ (error?: unknown) => (error ? reject(error) : resolve({ next: true })),
+ );
+ });
+ }
+
+ describe("scrape blocklist gate", () => {
+ it("lets an unblocked URL through even when its provider's terms are unaccepted", async () => {
+ await expect(
+ runGate({ url: OPEN_URL, formats: ["markdown"] }, UNACCEPTED_FLAGS),
+ ).resolves.toEqual({ next: true });
+ });
+
+ it("asks for the terms when they are all that keeps a blocked URL out", async () => {
+ const result = await runGate(
+ { url: BLOCKED_URL, formats: ["markdown"] },
+ UNACCEPTED_FLAGS,
+ );
+
+ expect(result).toEqual({
+ status: 403,
+ body: new exchange.ThirdPartyDataTermsRequiredError(TERMS).response(),
+ });
+ });
+
+ it("lets a blocked URL through when the Exchange can serve it", async () => {
+ await expect(
+ runGate({ url: BLOCKED_URL, formats: ["markdown"] }, ACCEPTED_FLAGS),
+ ).resolves.toEqual({ next: true });
+ });
+
+ it("blocks a blocked URL the Exchange cannot serve", async () => {
+ const result = await runGate(
+ {
+ url: BLOCKED_URL,
+ formats: ["markdown"],
+ actions: [{ type: "wait", milliseconds: 1000 }],
+ },
+ ACCEPTED_FLAGS,
+ );
+
+ expect(result).toMatchObject({
+ status: 403,
+ body: { success: false },
+ });
+ expect((result as any).body.code).toBeUndefined();
+ });
+ });
+
+ describe("buildFallbackList", () => {
+ it("routes to the Exchange when the terms are accepted", async () => {
+ const fallback = await buildFallbackList(
+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),
+ );
+
+ expect(fallback.map(f => f.engine)).toEqual(["exchange"]);
+ });
+
+ it("scrapes an unblocked URL normally when the terms are unaccepted", async () => {
+ const fallback = await buildFallbackList(
+ buildStubMeta(OPEN_URL, UNACCEPTED_FLAGS),
+ );
+ const engines = fallback.map(f => f.engine);
+
+ expect(engines).not.toContain("exchange");
+ expect(engines.length).toBeGreaterThan(0);
+ });
+
+ it("raises the terms error for a blocked URL when the terms are unaccepted", async () => {
+ const error = await buildFallbackList(
+ buildStubMeta(BLOCKED_URL, UNACCEPTED_FLAGS),
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(exchange.ThirdPartyDataTermsRequiredError);
+ expect(error.response()).toEqual(
+ new exchange.ThirdPartyDataTermsRequiredError(TERMS).response(),
+ );
+ });
+
+ it("fails closed for a blocked URL the Exchange cannot serve", async () => {
+ const meta = buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS);
+ meta.options.actions = [{ type: "wait", milliseconds: 1000 }];
+
+ await expect(buildFallbackList(meta)).resolves.toEqual([]);
+ });
+ });
+
+ describe("exchange engine", () => {
+ const failure = (code: string, message: string) => ({
+ success: false,
+ error: { code, message, retryable: false },
+ });
+
+ it("reports a missing record as THIRD_PARTY_DATA_NOT_FOUND", async () => {
+ mocks.robustFetch.mockResolvedValue(
+ failure(
+ "record_not_found",
+ "The requested Exchange record was not found.",
+ ),
+ );
+
+ const error = await scrapeURLWithExchange(
+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(scrapeErrors.ExchangeRefusedError);
+ expect(error.code).toBe("THIRD_PARTY_DATA_NOT_FOUND");
+ expect(error.message).toBe(
+ "The requested Exchange record was not found.",
+ );
+ });
+
+ it("reports a provider the team lacks as THIRD_PARTY_DATA_NOT_ENABLED", async () => {
+ mocks.robustFetch.mockResolvedValue(
+ failure(
+ "provider_not_enabled",
+ "Exchange provider acme is not enabled for this team.",
+ ),
+ );
+
+ const error = await scrapeURLWithExchange(
+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(scrapeErrors.ExchangeRefusedError);
+ expect(error.code).toBe("THIRD_PARTY_DATA_NOT_ENABLED");
+ expect(error.message).toBe(
+ "Exchange provider acme is not enabled for this team.",
+ );
+ });
+
+ it("keeps any other failure an engine error", async () => {
+ mocks.robustFetch.mockResolvedValue(
+ failure(
+ "internal_error",
+ "The Exchange request could not be completed.",
+ ),
+ );
+
+ const error = await scrapeURLWithExchange(
+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(scrapeErrors.EngineError);
+ });
+ });
+});
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index 9706c3d30..808f7646a 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -12,6 +12,7 @@ import { getCrawlJobAccess } from "../../lib/operational-job-access";
import { logger as _logger } from "../../lib/logger";
import { deserializeTransportableError } from "../../lib/error-serde";
import { TransportableError } from "../../lib/error";
+import { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";
import { scrapeQueue } from "../../services/worker/nuq-router";
configDotenv();
@@ -87,6 +88,9 @@ export async function crawlErrorsController(
? {
code: error.code,
error: error.message,
+ ...(error instanceof ThirdPartyDataTermsRequiredError
+ ? { requiresAction: error.requiresAction }
+ : {}),
}
: {
error: x.failedReason!,
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 5aa14a33d..8e4d30d77 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -12,6 +12,7 @@ import { v7 as uuidv7 } from "uuid";
import { getJobPriority } from "../../lib/job-priority";
import { fromV1ScrapeOptions } from "../v2/types";
import { TransportableError } from "../../lib/error";
+import { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";
import { NuQJob } from "../../services/worker/nuq";
import { checkPermissions } from "../../lib/permissions";
import {
@@ -410,6 +411,29 @@ async function scrapeControllerInner(
});
}
+ if (e instanceof ThirdPartyDataTermsRequiredError) {
+ return res.status(403).json(e.response());
+ }
+
+ if (
+ e.code === "THIRD_PARTY_DATA_NOT_ENABLED" ||
+ e.code === "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED"
+ ) {
+ return res.status(403).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
+ if (e.code === "THIRD_PARTY_DATA_NOT_FOUND") {
+ return res.status(404).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
return res.status(timeoutErr ? 408 : 500).json({
success: false,
code: e.code,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 52d617079..f80d7a0c1 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1,4 +1,5 @@
import type { AgentInteropStatus } from "../../lib/agent-interop";
+import type { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";
import { Request } from "express";
import { config } from "../../config";
import { z } from "zod";
@@ -1303,6 +1304,7 @@ export type CrawlErrorsResponse =
timestamp?: string;
url: string;
error: string;
+ requiresAction?: ThirdPartyDataTermsRequiredError["requiresAction"];
}[];
robotsBlocked: string[];
};
diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts
index 9706c3d30..808f7646a 100644
--- a/apps/api/src/controllers/v2/crawl-errors.ts
+++ b/apps/api/src/controllers/v2/crawl-errors.ts
@@ -12,6 +12,7 @@ import { getCrawlJobAccess } from "../../lib/operational-job-access";
import { logger as _logger } from "../../lib/logger";
import { deserializeTransportableError } from "../../lib/error-serde";
import { TransportableError } from "../../lib/error";
+import { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";
import { scrapeQueue } from "../../services/worker/nuq-router";
configDotenv();
@@ -87,6 +88,9 @@ export async function crawlErrorsController(
? {
code: error.code,
error: error.message,
+ ...(error instanceof ThirdPartyDataTermsRequiredError
+ ? { requiresAction: error.requiresAction }
+ : {}),
}
: {
error: x.failedReason!,
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 83af538d8..3884b47f7 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -17,6 +17,7 @@ import {
getTimeoutProcessingDetails,
TransportableError,
} from "../../lib/error";
+import { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";
import { NuQJob } from "../../services/worker/nuq";
import { checkPermissions } from "../../lib/permissions";
import {
@@ -593,6 +594,38 @@ export async function scrapeController(
});
}
+ if (e instanceof ThirdPartyDataTermsRequiredError) {
+ setSpanAttributes(span, {
+ "scrape.status_code": 403,
+ });
+ return res.status(403).json(e.response());
+ }
+
+ if (
+ e.code === "THIRD_PARTY_DATA_NOT_ENABLED" ||
+ e.code === "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED"
+ ) {
+ setSpanAttributes(span, {
+ "scrape.status_code": 403,
+ });
+ return res.status(403).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
+ if (e.code === "THIRD_PARTY_DATA_NOT_FOUND") {
+ setSpanAttributes(span, {
+ "scrape.status_code": 404,
+ });
+ return res.status(404).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
const statusCode = timeoutErr ? 408 : 500;
setSpanAttributes(span, {
"scrape.status_code": statusCode,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 3a958a0e5..20ce7703a 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1,4 +1,5 @@
import type { AgentInteropStatus } from "../../lib/agent-interop";
+import type { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";
import { Request, Response } from "express";
import { hasCategory } from "../../lib/search-query-builder";
import { config } from "../../config";
@@ -2013,6 +2014,7 @@ export type CrawlErrorsResponse =
url: string;
code?: ErrorCodes;
error: string;
+ requiresAction?: ThirdPartyDataTermsRequiredError["requiresAction"];
}[];
robotsBlocked: string[];
};
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 6033b4a25..556f58e09 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -42,13 +42,17 @@ import {
PromptInjectionDetectedError,
JsonExtractionContentTooLargeError,
XTwitterConfigurationError,
+ ExchangeRefusedError,
} from "../scraper/scrapeURL/error";
import { UnsafeDomainBlockedError } from "./threat-protection/error";
+import { ThirdPartyDataTermsRequiredError } from "./exchange";
// TODO: figure out correct typing for this
const errorMap: Record<ErrorCodes, any> = {
- // Terms responses are API-level, never transported through workers.
- THIRD_PARTY_DATA_TERMS_REQUIRED: null,
+ THIRD_PARTY_DATA_TERMS_REQUIRED: ThirdPartyDataTermsRequiredError,
+ THIRD_PARTY_DATA_NOT_FOUND: ExchangeRefusedError,
+ THIRD_PARTY_DATA_NOT_ENABLED: ExchangeRefusedError,
+ THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED: ExchangeRefusedError,
SAFE_MODE_BLOCKED: null,
SCRAPE_SITE_RESTRICTION_BLOCKED: SiteRestrictionError,
SCRAPE_TIMEOUT: ScrapeJobTimeoutError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 09394ba0b..b32d40f53 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -2,6 +2,9 @@ import { UNSUPPORTED_SITE_MESSAGE } from "./strings";
export type ErrorCodes =
| "THIRD_PARTY_DATA_TERMS_REQUIRED"
+ | "THIRD_PARTY_DATA_NOT_FOUND"
+ | "THIRD_PARTY_DATA_NOT_ENABLED"
+ | "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED"
| "SCRAPE_TIMEOUT"
| "MAP_TIMEOUT"
| "UNKNOWN_ERROR"
diff --git a/apps/api/src/lib/exchange-request.ts b/apps/api/src/lib/exchange-request.ts
index 17238155b..9c77bf952 100644
--- a/apps/api/src/lib/exchange-request.ts
+++ b/apps/api/src/lib/exchange-request.ts
@@ -1,11 +1,6 @@
-import {
- getExchangeAccessForRequest,
- type ExchangeAccess,
-} from "./exchange";
+import { getExchangeAccessForRequest, type ExchangeAccess } from "./exchange";
-type ExchangeFlags = Parameters<
- typeof getExchangeAccessForRequest
->[0]["flags"];
+type ExchangeFlags = Parameters<typeof getExchangeAccessForRequest>[0]["flags"];
/**
* Run the Exchange access gate against an API request body, sourcing the
@@ -17,7 +12,10 @@ export function getExchangeAccessForRequestBody(input: {
body: Record<string, any>;
flags: ExchangeFlags;
url: string;
+ blocked: boolean;
zeroDataRetention: boolean;
+ teamId: string | null;
+ orgId: string | null;
}): Promise<ExchangeAccess> {
const body = input.body ?? {};
const scrapeOptions =
@@ -27,6 +25,9 @@ export function getExchangeAccessForRequestBody(input: {
return getExchangeAccessForRequest({
url: input.url,
+ teamId: input.teamId,
+ orgId: input.orgId,
+ blocked: input.blocked,
formats: scrapeOptions.formats,
actions: scrapeOptions.actions,
headers: scrapeOptions.headers,
diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.ts
index cfdd098bc..92aee817f 100644
--- a/apps/api/src/lib/exchange.test.ts
+++ b/apps/api/src/lib/exchange.test.ts
@@ -2,6 +2,11 @@ import { fetch } from "undici";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { config } from "../config";
+import { exchangeRequest } from "../services/alexandria/client";
+import {
+ deserializeTransportableError,
+ serializeTransportableError,
+} from "./error-serde";
import {
canUseExchangeForRequest,
clearExchangeProvidersForTest,
@@ -10,17 +15,21 @@ import {
getExchangeRequestLogContext,
getExchangeResponseLogContext,
getExchangeSuccessCredits,
- getThirdPartyDataTermsRequiredResponse,
isSuccessfulExchangeStatusCode,
isSupportedExchangeFormatRequest,
resolveExchangeProvider,
setExchangeProvidersForTest,
+ ThirdPartyDataTermsRequiredError,
} from "./exchange";
vi.mock("undici", () => ({
fetch: vi.fn(),
}));
+vi.mock("../services/alexandria/client", () => ({
+ exchangeRequest: vi.fn(),
+}));
+
const originalConfig = {
FIRE_EXCHANGE_URL: config.FIRE_EXCHANGE_URL,
USE_DB_AUTHENTICATION: config.USE_DB_AUTHENTICATION,
@@ -290,7 +299,7 @@ describe("Exchange routing", () => {
expect(getExchangeResponseLogContext(null)).toEqual({});
});
- it("accepts only formats the Exchange can return directly", () => {
+ it("accepts only formats derivable from the Exchange's markdown", () => {
expect(isSupportedExchangeFormatRequest(undefined)).toBe(true);
expect(isSupportedExchangeFormatRequest([{ type: "markdown" }])).toBe(true);
expect(isSupportedExchangeFormatRequest(["json"])).toBe(true);
@@ -300,7 +309,10 @@ describe("Exchange routing", () => {
{ type: "json" },
]),
).toBe(true);
- expect(isSupportedExchangeFormatRequest([{ type: "html" }])).toBe(false);
+ expect(isSupportedExchangeFormatRequest([{ type: "html" }])).toBe(true);
+ expect(isSupportedExchangeFormatRequest([{ type: "screenshot" }])).toBe(
+ false,
+ );
// deterministicJson extractors run against page HTML, which Exchange
// responses do not carry.
expect(
@@ -401,7 +413,9 @@ describe("Exchange routing", () => {
terms: ACME_TERMS,
});
- const response = getThirdPartyDataTermsRequiredResponse(ACME_TERMS);
+ const response = new ThirdPartyDataTermsRequiredError(
+ ACME_TERMS,
+ ).response();
expect(response).toMatchObject({
success: false,
code: "THIRD_PARTY_DATA_TERMS_REQUIRED",
@@ -485,7 +499,7 @@ describe("Exchange routing", () => {
).resolves.toEqual({ allowed: false, termsRequired: false });
});
- it("does not route unless the beta flag is enabled", async () => {
+ it("requires the beta flag except for blocked URLs", async () => {
await expect(
canUseExchangeForRequest({
url: "https://profiles.example/person/example-person",
@@ -500,6 +514,27 @@ describe("Exchange routing", () => {
flags: { professionalProfileCompanyDataBeta: false },
}),
).resolves.toBe(false);
+
+ // A blocked URL has no other way to be served, and still needs its terms.
+ await expect(
+ canUseExchangeForRequest({
+ url: "https://facts.example/records/1",
+ formats: [{ type: "markdown" }],
+ waitFor: 1000,
+ blocked: true,
+ }),
+ ).resolves.toBe(true);
+ await expect(
+ getExchangeAccessForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ blocked: true,
+ }),
+ ).resolves.toEqual({
+ allowed: false,
+ termsRequired: true,
+ terms: ACME_TERMS,
+ });
});
it("does not route unless the Exchange is configured", async () => {
@@ -670,3 +705,264 @@ describe("Exchange routing", () => {
expect(fetch).toHaveBeenCalledTimes(1);
});
});
+
+describe("Exchange terms acceptance", () => {
+ const DIGEST = "a".repeat(64);
+ const PROFILE_REQUEST = {
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ teamId: "team-1",
+ orgId: "org-1",
+ };
+
+ // Answers the Exchange's requirements and ledger status routes the way
+ // the acceptance fallback reads them.
+ function ledgerAnswers(
+ ledger: {
+ version: string;
+ textHash: string;
+ acceptedAt?: string;
+ }[],
+ ) {
+ vi.mocked(exchangeRequest).mockImplementation(async ({ path }) => {
+ if (path === "/v1/provider-terms/requirements") {
+ return {
+ status: 200,
+ body: {
+ providers: [
+ {
+ provider: "acme",
+ required: true,
+ terms: { ...ACME_TERMS, digest: DIGEST },
+ },
+ ],
+ },
+ };
+ }
+ if (path.startsWith("/v1/provider-terms/status")) {
+ return {
+ status: 200,
+ body: {
+ providers: ledger.map(entry => ({
+ provider: "acme",
+ revoked: false,
+ acceptedAt: "2026-01-05T00:00:00.000Z",
+ ...entry,
+ })),
+ },
+ };
+ }
+ return { status: 404, body: {} };
+ });
+ }
+
+ beforeEach(() => {
+ vi.mocked(exchangeRequest).mockReset();
+ config.FIRE_EXCHANGE_URL = "https://exchange.example";
+ config.USE_DB_AUTHENTICATION = true;
+ setExchangeProvidersForTest(TEST_PROVIDERS);
+ });
+
+ afterEach(() => {
+ config.FIRE_EXCHANGE_URL = originalConfig.FIRE_EXCHANGE_URL;
+ config.USE_DB_AUTHENTICATION = originalConfig.USE_DB_AUTHENTICATION;
+ clearExchangeProvidersForTest();
+ });
+
+ it("routes when the terms were accepted on the Exchange ledger", async () => {
+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: DIGEST }]);
+
+ const access = await getExchangeAccessForRequest({
+ ...PROFILE_REQUEST,
+ flags: { professionalProfileCompanyDataBeta: true },
+ });
+
+ expect(access.allowed).toBe(true);
+ });
+
+ it("does not accept a ledger entry for different terms text", async () => {
+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: "b".repeat(64) }]);
+
+ await expect(
+ getExchangeAccessForRequest({
+ ...PROFILE_REQUEST,
+ flags: { professionalProfileCompanyDataBeta: true },
+ }),
+ ).resolves.toEqual({
+ allowed: false,
+ termsRequired: true,
+ terms: ACME_TERMS,
+ });
+ });
+
+ it("skips the ledger for requests without an organization", async () => {
+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: DIGEST }]);
+
+ const access = await getExchangeAccessForRequest({
+ ...PROFILE_REQUEST,
+ orgId: null,
+ flags: { professionalProfileCompanyDataBeta: true },
+ });
+
+ expect(access).toMatchObject({ allowed: false, termsRequired: true });
+ expect(exchangeRequest).not.toHaveBeenCalled();
+ });
+
+ it("reads as not accepted when the ledger is unreachable", async () => {
+ vi.mocked(exchangeRequest).mockRejectedValue(new Error("unreachable"));
+
+ await expect(
+ getExchangeAccessForRequest({
+ ...PROFILE_REQUEST,
+ flags: { professionalProfileCompanyDataBeta: true },
+ }),
+ ).resolves.toMatchObject({ allowed: false, termsRequired: true });
+ });
+
+ it("caches ledger answers per organization and provider", async () => {
+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: DIGEST }]);
+ const input = {
+ ...PROFILE_REQUEST,
+ flags: { professionalProfileCompanyDataBeta: true },
+ };
+
+ await getExchangeAccessForRequest(input);
+ await getExchangeAccessForRequest(input);
+
+ // One requirements read and one ledger read, not two of each.
+ expect(exchangeRequest).toHaveBeenCalledTimes(2);
+ });
+
+ it("does not share cached ledger answers between teams", async () => {
+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: DIGEST }]);
+ const flags = { professionalProfileCompanyDataBeta: true };
+
+ await getExchangeAccessForRequest({ ...PROFILE_REQUEST, flags });
+ await getExchangeAccessForRequest({
+ ...PROFILE_REQUEST,
+ teamId: "team-2",
+ flags,
+ });
+
+ expect(exchangeRequest).toHaveBeenCalledTimes(4);
+ });
+
+ it("rechecks the ledger once the catalog carries new terms", async () => {
+ const input = {
+ ...PROFILE_REQUEST,
+ flags: { professionalProfileCompanyDataBeta: true },
+ };
+ ledgerAnswers([{ version: ACME_TERMS.version, textHash: DIGEST }]);
+ await expect(getExchangeAccessForRequest(input)).resolves.toMatchObject({
+ allowed: true,
+ });
+
+ // The provider publishes new terms; the ledger still holds the old ones.
+ const NEW_TERMS = { key: "acme", version: "2026-02-01" };
+ setExchangeProvidersForTest(
+ TEST_PROVIDERS.map(provider =>
+ provider.id === "acme" ? { ...provider, terms: NEW_TERMS } : provider,
+ ),
+ );
+ vi.mocked(exchangeRequest).mockImplementation(async ({ path }) =>
+ path === "/v1/provider-terms/requirements"
+ ? {
+ status: 200,
+ body: {
+ providers: [
+ {
+ provider: "acme",
+ required: true,
+ terms: { ...NEW_TERMS, digest: "c".repeat(64) },
+ },
+ ],
+ },
+ }
+ : {
+ status: 200,
+ body: {
+ providers: [
+ {
+ provider: "acme",
+ revoked: false,
+ version: ACME_TERMS.version,
+ textHash: DIGEST,
+ acceptedAt: "2026-01-05T00:00:00.000Z",
+ },
+ ],
+ },
+ },
+ );
+
+ await expect(getExchangeAccessForRequest(input)).resolves.toEqual({
+ allowed: false,
+ termsRequired: true,
+ terms: NEW_TERMS,
+ });
+ });
+
+ it("asks for terms again after an admin revocation until re-accepted", async () => {
+ const revokedFlags = {
+ professionalProfileCompanyDataBeta: true,
+ organizationDataSourceAccess: {
+ acme: {
+ status: "disabled",
+ termsKey: "acme",
+ termsVersion: "2026-01-01",
+ disabledAt: "2026-01-10T00:00:00.000Z",
+ disabledReason: "revoked_by_organization_admin",
+ },
+ },
+ };
+
+ ledgerAnswers([
+ {
+ version: ACME_TERMS.version,
+ textHash: DIGEST,
+ acceptedAt: "2026-01-05T00:00:00.000Z",
+ },
+ ]);
+ await expect(
+ getExchangeAccessForRequest({ ...PROFILE_REQUEST, flags: revokedFlags }),
+ ).resolves.toEqual({
+ allowed: false,
+ termsRequired: true,
+ terms: ACME_TERMS,
+ });
+
+ clearExchangeProvidersForTest();
+ setExchangeProvidersForTest(TEST_PROVIDERS);
+ ledgerAnswers([
+ {
+ version: ACME_TERMS.version,
+ textHash: DIGEST,
+ acceptedAt: "2026-01-11T00:00:00.000Z",
+ },
+ ]);
+ const access = await getExchangeAccessForRequest({
+ ...PROFILE_REQUEST,
+ flags: revokedFlags,
+ });
+ expect(access.allowed).toBe(true);
+ });
+
+ it("carries the terms-required response through the worker queue", () => {
+ const error = deserializeTransportableError(
+ serializeTransportableError(
+ new ThirdPartyDataTermsRequiredError(ACME_TERMS),
+ ),
+ );
+
+ expect(error).toBeInstanceOf(ThirdPartyDataTermsRequiredError);
+ expect((error as ThirdPartyDataTermsRequiredError).response()).toEqual(
+ new ThirdPartyDataTermsRequiredError(ACME_TERMS).response(),
+ );
+ expect(error?.message).toBe(
+ new ThirdPartyDataTermsRequiredError(ACME_TERMS).response().error,
+ );
+ expect((error as ThirdPartyDataTermsRequiredError).requiresAction).toEqual(
+ new ThirdPartyDataTermsRequiredError(ACME_TERMS).response()
+ .requiresAction,
+ );
+ });
+});
diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.ts
index 937ddd8ac..7048bfdcf 100644
--- a/apps/api/src/lib/exchange.ts
+++ b/apps/api/src/lib/exchange.ts
@@ -3,6 +3,8 @@ import { z } from "zod";
import { config } from "../config";
import type { FormatObject } from "../controllers/v2/types";
+import { hasLedgerAcceptance } from "../services/alexandria/terms";
+import { type ErrorCodes, TransportableError } from "./error";
import { logger as rootLogger } from "./logger";
type OrganizationDataSourceAccessRecord = {
@@ -22,6 +24,10 @@ type OrganizationDataSourceAccess = Record<
type RouteInput = {
url: string;
+ teamId?: string | null;
+ orgId?: string | null;
+ /** The URL is on the team's blocklist, so the Exchange is the only way to serve it. */
+ blocked?: boolean;
formats?: FormatObject[] | unknown[];
actions?: unknown[];
headers?: Record<string, unknown>;
@@ -65,11 +71,22 @@ type ExchangeProvider = {
}[];
};
-// deterministicJson is deliberately unsupported: its extractor scripts run
-// against page HTML, which Exchange responses do not carry.
-const SUPPORTED_FORMATS = new Set(["markdown", "json"]);
+// Formats the regular transformers derive from the Exchange's markdown (and the
+// HTML rendered from it). deterministicJson, screenshots and the like need the
+// real page, which the Exchange never fetches.
+const SUPPORTED_FORMATS = new Set([
+ "markdown",
+ "html",
+ "rawHtml",
+ "links",
+ "images",
+ "json",
+ "summary",
+ "question",
+ "highlights",
+ "query",
+]);
const EXCHANGE_BETA_FLAG = "professionalProfileCompanyDataBeta";
-const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED";
const EXCHANGE_PROVIDERS_PATH = "/v1/providers";
const EXCHANGE_PROVIDERS_TIMEOUT_MS = 2_000;
@@ -238,9 +255,16 @@ function providerMatchesUrl(
const host = normalizeHost(parsed.hostname);
const pathname = parsed.pathname || "/";
+ // "*.example.com" claims every subdomain of example.com, never the apex.
+ const wildcards = [...host.matchAll(/\./g)].map(
+ dot => `*${host.slice(dot.index)}`,
+ );
return provider.routes.some(route => {
- if (!route.domains.has(host)) {
+ if (
+ !route.domains.has(host) &&
+ !wildcards.some(wildcard => route.domains.has(wildcard))
+ ) {
return false;
}
@@ -258,16 +282,33 @@ function providerMatchesUrl(
});
}
+// Organizations kept on the direct FullEnrich path for LinkedIn, which predates
+// enrichment preferences: they skip the enrichment provider and resolve to the
+// next claimant in the catalog.
+const ENRICHMENT_PROVIDER_ID = "firecrawl-enrich";
+const LEGACY_FULLENRICH_ORGS = new Set([
+ "34a599c6-e6c2-4e6f-b563-0e23bb2552c1",
+ "adcef175-be20-4534-96ad-fcd413e3c457",
+ "2567598d-d959-44d3-8c4a-19fd467ec66d",
+ "709cf0a7-7769-4c7e-a5e1-ad44f38f9c36",
+]);
+
export async function resolveExchangeProvider(
inputUrl: string,
+ orgId?: string | null,
): Promise<ExchangeProvider | null> {
const providers = await getExchangeProviders();
if (providers === null) {
return null;
}
+ const skipEnrichment = orgId != null && LEGACY_FULLENRICH_ORGS.has(orgId);
return (
- providers.find(provider => providerMatchesUrl(provider, inputUrl)) ?? null
+ providers.find(
+ provider =>
+ !(skipEnrichment && provider.id === ENRICHMENT_PROVIDER_ID) &&
+ providerMatchesUrl(provider, inputUrl),
+ ) ?? null
);
}
@@ -356,15 +397,80 @@ export function isSupportedExchangeFormatRequest(
type DataSourceAccessDecision = "allowed" | "terms_required" | "not_enabled";
-function getProviderAccessDecision(
+const LEDGER_ACCEPTANCE_TIMEOUT_MS = 2_000;
+const LEDGER_ACCEPTED_TTL_MS = 60_000;
+const LEDGER_NOT_ACCEPTED_TTL_MS = 10_000;
+const LEDGER_ACCEPTANCE_CACHE_MAX_ENTRIES = 10_000;
+
+const ledgerAcceptanceCache = new Map<
+ string,
+ { expiresAt: number; value: Promise<boolean> }
+>();
+
+// The ledger lookup costs two Exchange calls and sits on the scrape path, so
+// answers are cached per process: briefly when not accepted, so a fresh
+// acceptance is picked up quickly, and longer once accepted. Keyed by the
+// team the Exchange calls are made for and the catalog's terms identity, so
+// new terms are rechecked as soon as the catalog carries them.
+function getLedgerAcceptance(input: {
+ teamId: string;
+ orgId: string;
+ provider: string;
+ terms: ExchangeTerms;
+ revocation?: { disabledAt: unknown };
+}): Promise<boolean> {
+ const key = [
+ input.teamId,
+ input.orgId,
+ input.provider,
+ input.terms.key,
+ input.terms.version,
+ input.revocation === undefined ? "" : String(input.revocation.disabledAt),
+ ].join("\0");
+ const cached = ledgerAcceptanceCache.get(key);
+ if (cached && cached.expiresAt > Date.now()) {
+ return cached.value;
+ }
+
+ if (ledgerAcceptanceCache.size >= LEDGER_ACCEPTANCE_CACHE_MAX_ENTRIES) {
+ ledgerAcceptanceCache.clear();
+ }
+
+ const entry = {
+ expiresAt: Date.now() + LEDGER_NOT_ACCEPTED_TTL_MS,
+ value: Promise.resolve(false),
+ };
+ entry.value = hasLedgerAcceptance({
+ teamId: input.teamId,
+ orgId: input.orgId,
+ provider: input.provider,
+ revocation: input.revocation,
+ timeoutMs: LEDGER_ACCEPTANCE_TIMEOUT_MS,
+ })
+ .catch(() => false)
+ .then(accepted => {
+ if (accepted) {
+ entry.expiresAt = Date.now() + LEDGER_ACCEPTED_TTL_MS;
+ }
+ return accepted;
+ });
+ ledgerAcceptanceCache.set(key, entry);
+ return entry.value;
+}
+
+// Mirrors authorizeProviders (services/alexandria/access.ts), so a provider
+// is reachable through Scrape exactly when it is through Alexandria: the
+// organizationDataSourceAccess flags first, then the Exchange ledger, where
+// the API's accept route records acceptance.
+async function getProviderAccessDecision(
provider: ExchangeProvider,
- flags: RouteInput["flags"],
-): DataSourceAccessDecision {
+ input: RouteInput,
+): Promise<DataSourceAccessDecision> {
if (config.USE_DB_AUTHENTICATION !== true) {
return "allowed";
}
- const access = flags?.organizationDataSourceAccess?.[provider.id];
+ const access = input.flags?.organizationDataSourceAccess?.[provider.id];
const entry = typeof access === "object" && access !== null ? access : null;
if (provider.terms === undefined) {
@@ -373,22 +479,58 @@ function getProviderAccessDecision(
: "allowed";
}
- if (entry === null) {
- return "terms_required";
- }
+ const teamId = input.teamId ?? null;
+ const orgId = input.orgId ?? null;
+
+ if (entry !== null && entry.status !== "enabled") {
+ // An admin revocation is lifted by a ledger acceptance recorded after
+ // it; until then the provider asks for its terms again.
+ const revokedByAdmin =
+ entry.status === "disabled" &&
+ entry.disabledReason === "revoked_by_organization_admin";
+ if (!revokedByAdmin || teamId === null || orgId === null) {
+ return "not_enabled";
+ }
- if (entry.status !== "enabled") {
- return "not_enabled";
+ return (await getLedgerAcceptance({
+ teamId,
+ orgId,
+ provider: provider.id,
+ terms: provider.terms,
+ revocation: { disabledAt: entry.disabledAt },
+ }))
+ ? "allowed"
+ : "terms_required";
}
- return entry.termsKey === provider.terms.key &&
+ if (
+ entry !== null &&
+ entry.termsKey === provider.terms.key &&
entry.termsVersion === provider.terms.version
- ? "allowed"
- : "terms_required";
+ ) {
+ return "allowed";
+ }
+
+ if (
+ teamId !== null &&
+ orgId !== null &&
+ (await getLedgerAcceptance({
+ teamId,
+ orgId,
+ provider: provider.id,
+ terms: provider.terms,
+ }))
+ ) {
+ return "allowed";
+ }
+
+ return "terms_required";
}
function isExchangeEligibleRequest(input: RouteInput): boolean {
- if (input.flags?.[EXCHANGE_BETA_FLAG] !== true) {
+ // Blocked URLs go to the Exchange for every team, since nothing else may
+ // serve them; elsewhere it replaces a normal scrape only for the beta.
+ if (input.flags?.[EXCHANGE_BETA_FLAG] !== true && input.blocked !== true) {
return false;
}
@@ -408,27 +550,28 @@ function isExchangeEligibleRequest(input: RouteInput): boolean {
return false;
}
- if (input.headers && Object.keys(input.headers).length > 0) {
- return false;
- }
-
- if (input.waitFor !== undefined && input.waitFor !== 0) {
- return false;
- }
-
- if (input.mobile || input.location || input.blockAds === false) {
- return false;
- }
-
// Profile-backed scrapes expect session-specific content, which the
// Exchange cannot serve.
if (input.profile !== undefined) {
return false;
}
- // atsv is only supported by browser engines; requests that set it keep an
- // engine that can honor it instead of routing to the Exchange.
- if (input.atsv === true) {
+ // Rendering options only mean something for a real page. A blocked URL has
+ // no page Firecrawl may render, so they are ignored there; anywhere else a
+ // request that sets them keeps the normal engines.
+ if (
+ input.blocked !== true &&
+ ((input.headers !== undefined && Object.keys(input.headers).length > 0) ||
+ (input.waitFor !== undefined && input.waitFor !== 0) ||
+ input.mobile ||
+ input.location ||
+ input.blockAds === false ||
+ input.atsv === true ||
+ input.proxy === "stealth" ||
+ input.proxy === "enhanced" ||
+ (Array.isArray(input.includeTags) && input.includeTags.length > 0) ||
+ (Array.isArray(input.excludeTags) && input.excludeTags.length > 0))
+ ) {
return false;
}
@@ -439,18 +582,6 @@ function isExchangeEligibleRequest(input: RouteInput): boolean {
return false;
}
- // Selector-based content filtering does not apply to provider records.
- if (
- (Array.isArray(input.includeTags) && input.includeTags.length > 0) ||
- (Array.isArray(input.excludeTags) && input.excludeTags.length > 0)
- ) {
- return false;
- }
-
- if (input.proxy === "stealth" || input.proxy === "enhanced") {
- return false;
- }
-
if (!isSupportedExchangeFormatRequest(input.formats)) {
return false;
}
@@ -485,12 +616,12 @@ export async function getExchangeAccessForRequest(
return { allowed: false, termsRequired: false };
}
- const provider = await resolveExchangeProvider(input.url);
+ const provider = await resolveExchangeProvider(input.url, input.orgId);
if (provider === null) {
return { allowed: false, termsRequired: false };
}
- const decision = getProviderAccessDecision(provider, input.flags);
+ const decision = await getProviderAccessDecision(provider, input);
if (decision === "terms_required" && provider.terms !== undefined) {
return { allowed: false, termsRequired: true, terms: provider.terms };
}
@@ -518,19 +649,70 @@ function getThirdPartyDataTermsUrl(terms: ExchangeTerms): string {
return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/alexandria/${encodeURIComponent(terms.key)}`;
}
-export function getThirdPartyDataTermsRequiredResponse(terms: ExchangeTerms) {
- const url = getThirdPartyDataTermsUrl(terms);
- return {
- success: false as const,
- code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE as "THIRD_PARTY_DATA_TERMS_REQUIRED",
- error: `An organization admin must accept the ${terms.key} provider's terms (version ${terms.version}) before this request can run. Accept them at ${url}`,
- requiresAction: {
- type: "accept_terms",
- terms: terms.key,
- version: terms.version,
- url,
- },
- };
+export function getEnrichmentSettingsUrl(): string {
+ return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/alexandria?enrichment=true`;
+}
+
+/**
+ * An organization admin has to accept a provider's terms before the request
+ * can run. Transportable, so it crosses the worker queue intact; every
+ * surface that reports it sends `response()`, or `requiresAction` alone where
+ * the error is one entry of a list.
+ */
+export class ThirdPartyDataTermsRequiredError extends TransportableError {
+ public readonly terms: ExchangeTerms;
+ /** Set when the provider is a step in the team's enrichment order, which can drop it instead. */
+ public readonly enrichment: boolean;
+
+ constructor(terms: ExchangeTerms, options: { enrichment?: boolean } = {}) {
+ const accept = `An organization admin must accept the ${terms.key} provider's terms (version ${terms.version}) before this request can run. Accept them at ${getThirdPartyDataTermsUrl(terms)}`;
+ super(
+ "THIRD_PARTY_DATA_TERMS_REQUIRED",
+ options.enrichment
+ ? `${accept}, or turn off ${terms.key} in ${getEnrichmentSettingsUrl()}.`
+ : accept,
+ );
+ this.name = "ThirdPartyDataTermsRequiredError";
+ this.terms = { key: terms.key, version: terms.version };
+ this.enrichment = options.enrichment === true;
+ }
+
+ get requiresAction() {
+ return {
+ type: "accept_terms" as const,
+ terms: this.terms.key,
+ version: this.terms.version,
+ url: getThirdPartyDataTermsUrl(this.terms),
+ };
+ }
+
+ response() {
+ return {
+ success: false as const,
+ code: "THIRD_PARTY_DATA_TERMS_REQUIRED" as const,
+ error: this.message,
+ requiresAction: this.requiresAction,
+ };
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ terms: this.terms,
+ enrichment: this.enrichment,
+ };
+ }
+
+ static deserialize(
+ _code: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new ThirdPartyDataTermsRequiredError(data.terms, {
+ enrichment: data.enrichment,
+ });
+ x.stack = data.stack;
+ return x;
+ }
}
export function getExchangeSuccessCredits(input: {
@@ -759,4 +941,5 @@ export function setExchangeProvidersForTest(
export function clearExchangeProvidersForTest() {
cachedProviders = undefined;
providersRequest = undefined;
+ ledgerAcceptanceCache.clear();
}
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts
index 6ca688fc0..b9c1e690d 100644
--- a/apps/api/src/lib/scrape-billing.test.ts
+++ b/apps/api/src/lib/scrape-billing.test.ts
@@ -38,6 +38,34 @@ describe("calculateCreditsToBeBilled", () => {
expect(credits).toBe(12);
});
+ it("adds format surcharges on top of an Exchange access", async () => {
+ const credits = await calculateCreditsToBeBilled(
+ {
+ formats: [{ type: "json", prompt: "name" }],
+ } as any,
+ {
+ teamId: "team-id",
+ orgId: null,
+ },
+ {
+ metadata: {
+ statusCode: 200,
+ url: "https://profiles.example/person/example-person",
+ proxyUsed: "basic",
+ },
+ } as any,
+ {
+ totalCost: 0,
+ } as any,
+ {} as any,
+ undefined,
+ undefined,
+ { handled: true, creditsCost: 12 },
+ );
+
+ expect(credits).toBe(16);
+ });
+
it("bills X/Twitter scrapes at 30 credits", async () => {
const credits = await calculateCreditsToBeBilled(
{
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 0ec7e8733..2de29a4a5 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -153,15 +153,13 @@ export async function calculateCreditsToBeBilled(
return creditsToBeBilled + threatScanCredits;
}
+ // An Exchange access is priced by its provider in place of the base
+ // credit; format surcharges (json, question, ...) still apply on top.
const exchangeCredits = getExchangeSuccessCredits({
exchange,
statusCode: document.metadata?.statusCode,
});
- if (exchangeCredits !== null) {
- return exchangeCredits + threatScanCredits;
- }
-
- let creditsToBeBilled = 1; // Assuming 1 credit per document
+ let creditsToBeBilled = exchangeCredits ?? 1;
if (options.lockdown) {
creditsToBeBilled += 4;
@@ -202,7 +200,9 @@ export async function calculateCreditsToBeBilled(
internalOptions.v1Agent?.model === "fire-1" ||
internalOptions.v1JSONAgent?.model?.toLowerCase() === "fire-1"
) {
- creditsToBeBilled = Math.ceil((costTrackingJSON.totalCost ?? 1) * 1800);
+ creditsToBeBilled =
+ (exchangeCredits ?? 0) +
+ Math.ceil((costTrackingJSON.totalCost ?? 1) * 1800);
}
const hasQuestionFormat =
@@ -258,7 +258,10 @@ export async function calculateCreditsToBeBilled(
document.metadata?.url,
document.metadata?.sourceURL,
].filter((u): u is string => !!u);
- if (urlsToCheck.some(u => isUrlBlocked(u, null) && !isUrlBlocked(u, flags))) {
+ if (
+ exchangeCredits === null &&
+ urlsToCheck.some(u => isUrlBlocked(u, null) && !isUrlBlocked(u, flags))
+ ) {
creditsToBeBilled += unblockedDomainCostBonus;
}
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index b9f8eb78e..ec3253280 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -29,10 +29,11 @@ import {
CREDITS_FEATURE_ID,
} from "../services/autumn/autumn.service";
import { getTeamBalance } from "../services/autumn/usage";
-import { getThirdPartyDataTermsRequiredResponse } from "../lib/exchange";
+import { ThirdPartyDataTermsRequiredError } from "../lib/exchange";
import { getExchangeAccessForRequestBody } from "../lib/exchange-request";
import { isToolsOnlySearch } from "../search/alexandria";
import { getScrapeZDR } from "../lib/zdr-helpers";
+import { isLockdownZeroDataRetention } from "../lib/safe-mode";
import {
agentInteropStatus,
isAgentInteropSecretValid,
@@ -363,11 +364,14 @@ export function blocklistMiddleware(
}
/**
- * Blocklist gate for single-URL scrape-shaped routes (scrape, crawl), where
- * an Exchange-eligible URL may bypass the blocklist because the exchange
- * engine can serve it. Everything else (map, search, batch scrape, monitors)
- * keeps plain blocklist behavior - batch stays out until its jobs carry the
- * access flags the worker-side recheck needs.
+ * Blocklist gate for single-URL scrape-shaped routes (scrape, crawl), where a
+ * blocklisted URL still passes when the exchange engine can serve it, and
+ * asks for the provider's terms when unaccepted terms are all that stands in
+ * the way. Unblocked URLs never consult the Exchange here: engine selection
+ * decides for them, identically on every route, and scrapes them normally
+ * when terms are missing. Everything else (map, search, batch scrape,
+ * monitors) keeps plain blocklist behavior - batch stays out until its jobs
+ * carry the access flags the worker-side recheck needs.
*/
export function scrapeBlocklistMiddleware(
req: RequestWithMaybeACUC<any, any, any>,
@@ -384,44 +388,53 @@ function blocklistGate(
options: { exchange: boolean },
) {
(async () => {
- const zeroDataRetention =
- getScrapeZDR(req.acuc?.flags) === "forced" ||
- req.body?.zeroDataRetention === true;
- const exchangeAccess =
- options.exchange &&
- typeof req.body.url === "string" &&
- (await getExchangeAccessForRequestBody({
+ if (
+ typeof req.body.url !== "string" ||
+ !isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {
+ team_id: req.acuc?.team_id ?? null,
+ org_id: req.acuc?.org_id ?? null,
+ origin: typeof req.body.origin === "string" ? req.body.origin : null,
+ })
+ ) {
+ return next();
+ }
+
+ if (options.exchange) {
+ // Safe Mode lockdown implies ZDR, which keeps the Exchange out.
+ const zeroDataRetention =
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ req.body?.zeroDataRetention === true ||
+ isLockdownZeroDataRetention(req.acuc?.flags, req.body?.safeMode);
+ const exchangeAccess = await getExchangeAccessForRequestBody({
body: req.body,
flags: req.acuc?.flags ?? null,
url: req.body.url,
+ blocked: true,
zeroDataRetention,
- }));
- const canUseExchange =
- typeof exchangeAccess === "object" && exchangeAccess.allowed;
+ teamId: req.acuc?.team_id ?? null,
+ orgId: req.acuc?.org_id ?? null,
+ });
- if (typeof exchangeAccess === "object" && exchangeAccess.termsRequired) {
- if (!res.headersSent) {
+ if (exchangeAccess.allowed) {
+ return next();
+ }
+
+ if (exchangeAccess.termsRequired && !res.headersSent) {
return res
.status(403)
- .json(getThirdPartyDataTermsRequiredResponse(exchangeAccess.terms));
+ .json(
+ new ThirdPartyDataTermsRequiredError(
+ exchangeAccess.terms,
+ ).response(),
+ );
}
}
- if (
- typeof req.body.url === "string" &&
- !canUseExchange &&
- isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {
- team_id: req.acuc?.team_id ?? null,
- org_id: req.acuc?.org_id ?? null,
- origin: typeof req.body.origin === "string" ? req.body.origin : null,
- })
- ) {
- if (!res.headersSent) {
- return res.status(403).json({
- success: false,
- error: UNSUPPORTED_SITE_MESSAGE,
- });
- }
+ if (!res.headersSent) {
+ return res.status(403).json({
+ success: false,
+ error: UNSUPPORTED_SITE_MESSAGE,
+ });
}
next();
})().catch(err => next(err));
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index 04a758b55..b426fe959 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -17,6 +17,8 @@ type BlockContext = {
team_id?: string | null;
org_id?: string | null;
origin?: string | null;
+ /** False for a re-check of a request whose hit was already recorded. */
+ record?: boolean;
};
type BlockHit = {
@@ -56,7 +58,7 @@ function recordHit(
domain: string,
context: BlockContext | undefined,
): void {
- if (context === undefined) return;
+ if (context === undefined || context.record === false) return;
if (config.USE_DB_AUTHENTICATION !== true) return;
hitBuffer.push({
id: uuidv7(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/exchange.ts b/apps/api/src/scraper/scrapeURL/engines/exchange.ts
index c6c41de2d..60239dbfc 100644
--- a/apps/api/src/scraper/scrapeURL/engines/exchange.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/exchange.ts
@@ -5,11 +5,14 @@ import { EngineScrapeResult } from ".";
import { config } from "../../../config";
import {
getExchangeRequestLogContext,
+ getEnrichmentSettingsUrl,
getExchangeResponseLogContext,
+ ThirdPartyDataTermsRequiredError,
} from "../../../lib/exchange";
import { setSpanAttributes, withSpan } from "../../../lib/otel-tracer";
import { robustFetch } from "../lib/fetch";
-import { EngineError } from "../error";
+import { safeMarkdownToHtml } from "./pdf/markdownToHtml";
+import { EngineError, ExchangeRefusedError } from "../error";
const exchangeScrapeResponseSchema = z.union([
z
@@ -32,7 +35,6 @@ const exchangeScrapeResponseSchema = z.union([
.optional(),
metadata: z.record(z.string(), z.unknown()).optional(),
markdown: z.string().optional(),
- json: z.unknown().optional(),
})
.passthrough(),
})
@@ -44,6 +46,7 @@ const exchangeScrapeResponseSchema = z.union([
.object({
code: z.string().optional(),
message: z.string().optional(),
+ terms: z.object({ key: z.string(), version: z.string() }).optional(),
})
.passthrough()
.optional(),
@@ -51,6 +54,50 @@ const exchangeScrapeResponseSchema = z.union([
.passthrough(),
]);
+// Exchange refusals that describe the request itself. Any other failure is an
+// engine failure. Enrichment refusals point at the settings that fix them.
+const EXCHANGE_REFUSALS = new Map<
+ string,
+ {
+ code: ConstructorParameters<typeof ExchangeRefusedError>[0];
+ message: string;
+ enrichmentSettings?: true;
+ }
+>([
+ [
+ "record_not_found",
+ {
+ code: "THIRD_PARTY_DATA_NOT_FOUND",
+ message:
+ "The third-party data provider for this URL has no record for it.",
+ },
+ ],
+ [
+ "provider_not_enabled",
+ {
+ code: "THIRD_PARTY_DATA_NOT_ENABLED",
+ message:
+ "The third-party data provider for this URL is not enabled for this team.",
+ },
+ ],
+ [
+ "enrichment_not_enabled",
+ {
+ code: "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED",
+ message: "Enrichment is not enabled for this kind of profile.",
+ enrichmentSettings: true,
+ },
+ ],
+ [
+ "enrichment_unavailable",
+ {
+ code: "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED",
+ message: "None of the team's enrichment providers can serve this URL.",
+ enrichmentSettings: true,
+ },
+ ],
+]);
+
export function exchangeMaxReasonableTime(meta: Meta): number {
return meta.options.timeout ?? 60_000;
}
@@ -63,15 +110,21 @@ function escapeHtml(value: string): string {
.replace(/"/g, "&quot;");
}
-// Exchange responses carry no page HTML; synthesize a minimal head so the
-// metadata transformer can populate the document's title and description.
-function buildMetadataHtml(title?: string, description?: string): string {
- const titleTag = title === undefined ? "" : `<title>${escapeHtml(title)}</title>`;
+// Exchange responses carry no page HTML. Render the markdown into a page so
+// the regular transformers derive html, rawHtml, links, images and metadata
+// from it the way they do for any other page.
+function buildPageHtml(
+ body: string,
+ title?: string,
+ description?: string,
+): string {
+ const titleTag =
+ title === undefined ? "" : `<title>${escapeHtml(title)}</title>`;
const descriptionTag =
description === undefined
? ""
: `<meta name="description" content="${escapeHtml(description)}">`;
- return `<!DOCTYPE html><html><head>${titleTag}${descriptionTag}</head><body></body></html>`;
+ return `<!DOCTYPE html><html><head>${titleTag}${descriptionTag}</head><body>${body}</body></html>`;
}
export async function scrapeURLWithExchange(
@@ -105,10 +158,26 @@ export async function scrapeURLWithExchange(
requestId: meta.id,
teamId: meta.internalOptions.teamId,
url,
- formats: ["markdown", "json"],
+ // Named so the Exchange serves the provider whose access was checked
+ // when several claim the URL.
+ ...(meta.exchangeProviderId === undefined
+ ? {}
+ : { provider: meta.exchangeProviderId }),
+ formats: ["markdown"],
...(meta.options.maxAge === undefined
? {}
: { maxAge: meta.options.maxAge }),
+ // Enrichment checks each provider step's terms against these rows
+ // and the Exchange's own ledger for the organization.
+ ...(meta.internalOptions.orgId
+ ? { organizationId: meta.internalOptions.orgId }
+ : {}),
+ ...(meta.internalOptions.teamFlags?.organizationDataSourceAccess
+ ? {
+ organizationDataSourceAccess:
+ meta.internalOptions.teamFlags.organizationDataSourceAccess,
+ }
+ : {}),
},
logger: logger.child({ method: "exchangeScrape/robustFetch" }),
tryCount: 2,
@@ -126,6 +195,26 @@ export async function scrapeURLWithExchange(
errorCode: response.error?.code,
durationMs: Date.now() - startTime,
});
+ if (
+ response.error?.code === "third_party_data_terms_required" &&
+ response.error.terms !== undefined
+ ) {
+ // Only enrichment checks terms inside the Exchange, so the step can
+ // also be turned off in the team's enrichment settings.
+ throw new ThirdPartyDataTermsRequiredError(response.error.terms, {
+ enrichment: true,
+ });
+ }
+ const refusal = EXCHANGE_REFUSALS.get(response.error?.code ?? "");
+ if (refusal !== undefined) {
+ const message = response.error?.message || refusal.message;
+ throw new ExchangeRefusedError(
+ refusal.code,
+ refusal.enrichmentSettings
+ ? `${message} An organization admin can choose enrichment providers at ${getEnrichmentSettingsUrl()}`
+ : message,
+ );
+ }
throw new EngineError("Exchange request failed");
}
@@ -152,11 +241,15 @@ export async function scrapeURLWithExchange(
"exchange.duration_ms": Date.now() - startTime,
});
+ const markdown = response.data.markdown ?? "";
return {
url: response.data.url ?? url,
- html: buildMetadataHtml(response.data.title, response.data.description),
- markdown: response.data.markdown,
- json: response.data.json,
+ html: buildPageHtml(
+ await safeMarkdownToHtml(markdown, meta.logger, meta.id),
+ response.data.title,
+ response.data.description,
+ ),
+ markdown,
statusCode: 200,
contentType: "text/markdown",
proxyUsed: "basic",
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index e96279522..b9d7c4db2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -34,7 +34,6 @@ const successSchema = z.object({
// timeTaken: z.number(),
content: z.string(),
- json: z.unknown().optional(),
url: z.string().optional(),
pageStatusCode: z.number(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 4c533e830..33a60027f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -597,7 +597,6 @@ export async function scrapeURLWithFireEngineChromeCDP(
markdown: contentType?.includes("text/markdown")
? response.content
: undefined,
- json: response.json,
error: response.pageError,
statusCode: response.pageStatusCode,
@@ -698,7 +697,6 @@ export async function scrapeURLWithFireEngineTLSClient(
markdown: contentType?.includes("text/markdown")
? response.content
: undefined,
- json: response.json,
error: response.pageError,
statusCode: response.pageStatusCode,
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index d628a23c5..1f5b69927 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -112,7 +112,6 @@ const successSchema = z.object({
timeTaken: z.number(),
content: z.string(),
- json: z.unknown().optional(),
url: z.string().optional(),
pageStatusCode: z.number(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index d2207668b..fb38ebf31 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -44,7 +44,8 @@ import {
import { isUrlBlocked } from "../../WebScraper/utils/blocklist";
import { hasCustomRequestContext } from "../lib/request-context";
import {
- canUseExchangeForRequest,
+ getExchangeAccessForRequest,
+ ThirdPartyDataTermsRequiredError,
type ExchangeScrapeMetadata,
} from "../../../lib/exchange";
@@ -156,7 +157,6 @@ export type EngineScrapeResult = {
markdown?: string;
pages?: Array<{ pageNumber: number; markdown: string }>;
blocks?: PdfPageBlocks[];
- json?: unknown;
statusCode: number;
error?: string;
@@ -665,27 +665,54 @@ export async function buildFallbackList(meta: Meta): Promise<
!meta.internalOptions.agentIndexOnly &&
meta.internalOptions.forceEngine === undefined
) {
- if (
- await canUseExchangeForRequest({
- url: meta.rewrittenUrl ?? meta.url,
- formats: meta.options.formats,
- actions: meta.options.actions,
- headers: meta.options.headers,
- waitFor: meta.options.waitFor,
- mobile: meta.options.mobile,
- location: meta.options.location,
- proxy: meta.options.proxy,
- blockAds: meta.options.blockAds,
- profile: meta.options.profile,
- atsv: meta.internalOptions.atsv,
- minAge: meta.options.minAge,
- includeTags: meta.options.includeTags,
- excludeTags: meta.options.excludeTags,
- zeroDataRetention: meta.internalOptions.zeroDataRetention,
- lockdown: meta.options.lockdown,
- flags: meta.internalOptions.teamFlags ?? null,
- })
- ) {
+ const url = meta.rewrittenUrl ?? meta.url;
+ const betaTeam =
+ meta.internalOptions.teamFlags?.professionalProfileCompanyDataBeta ===
+ true;
+ let blocked = false;
+ try {
+ blocked = isUrlBlocked(url, meta.internalOptions.teamFlags ?? null, {
+ team_id: meta.internalOptions.teamId ?? null,
+ org_id: meta.internalOptions.orgId ?? null,
+ origin: null,
+ record: false,
+ });
+ } catch (error) {
+ // Beta teams fail closed, as they always have. Anyone else keeps the
+ // normal engines, since an unreadable blocklist must not fail every
+ // scrape.
+ if (betaTeam) {
+ meta.logger.warn("Exchange blocklist check failed; failing closed", {
+ error,
+ });
+ return [];
+ }
+ }
+
+ const exchangeAccess = await getExchangeAccessForRequest({
+ url,
+ teamId: meta.internalOptions.teamId ?? null,
+ orgId: meta.internalOptions.orgId ?? null,
+ blocked,
+ formats: meta.options.formats,
+ actions: meta.options.actions,
+ headers: meta.options.headers,
+ waitFor: meta.options.waitFor,
+ mobile: meta.options.mobile,
+ location: meta.options.location,
+ proxy: meta.options.proxy,
+ blockAds: meta.options.blockAds,
+ profile: meta.options.profile,
+ atsv: meta.internalOptions.atsv,
+ minAge: meta.options.minAge,
+ includeTags: meta.options.includeTags,
+ excludeTags: meta.options.excludeTags,
+ zeroDataRetention: meta.internalOptions.zeroDataRetention,
+ lockdown: meta.options.lockdown,
+ flags: meta.internalOptions.teamFlags ?? null,
+ });
+ if (exchangeAccess.allowed) {
+ meta.exchangeProviderId = exchangeAccess.provider.id;
return [
{
engine: "exchange",
@@ -694,37 +721,16 @@ export async function buildFallbackList(meta: Meta): Promise<
];
}
- // A blocked URL can only have been admitted by the Exchange bypass in
- // blocklistMiddleware, which only applies to flagged orgs; if the
- // Exchange is no longer usable by execution time (catalog changed,
- // service down), fail closed rather than letting normal engines scrape
- // a blocklisted site. An error here also fails closed: this branch only
- // runs for flagged orgs, and a retryable scrape failure is preferable
- // to scraping a potentially blocklisted site with normal engines.
- if (
- meta.internalOptions.teamFlags?.professionalProfileCompanyDataBeta ===
- true
- ) {
- try {
- if (
- isUrlBlocked(
- meta.rewrittenUrl ?? meta.url,
- meta.internalOptions.teamFlags ?? null,
- {
- team_id: meta.internalOptions.teamId ?? null,
- org_id: meta.internalOptions.orgId ?? null,
- origin: null,
- },
- )
- ) {
- return [];
- }
- } catch (error) {
- meta.logger.warn("Exchange blocklist re-check failed; failing closed", {
- error,
- });
- return [];
+ // A blocked URL can only have been admitted by scrapeBlocklistMiddleware
+ // for the Exchange; if the Exchange can no longer serve it (catalog
+ // changed, service down), fail closed rather than letting normal engines
+ // scrape a blocklisted site. Unblocked URLs whose provider wants
+ // unaccepted terms fall through and scrape normally.
+ if (blocked) {
+ if (exchangeAccess.termsRequired) {
+ throw new ThirdPartyDataTermsRequiredError(exchangeAccess.terms);
}
+ return [];
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index d2e652bb1..15c065291 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -120,7 +120,6 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
meta.rewrittenUrl ??
meta.url,
html: document.rawHtml!,
- json: document.json,
statusCode: document.metadata.statusCode,
error: document.metadata.error,
screenshot: document.screenshot,
@@ -588,7 +587,6 @@ export async function scrapeURLWithIndex(
return {
url: doc.url,
html: doc.html,
- json: doc.json,
statusCode: doc.statusCode,
error: doc.error,
screenshot: doc.screenshot,
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 19129fdb7..59bc1beec 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -121,6 +121,32 @@ export class SSLError extends TransportableError {
}
}
+type ExchangeRefusalCode =
+ | "THIRD_PARTY_DATA_NOT_FOUND"
+ | "THIRD_PARTY_DATA_NOT_ENABLED"
+ | "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED";
+
+// A definitive answer from the Exchange about this URL: the provider holds no
+// record for it, or the team is not entitled to the provider. Another attempt
+// cannot change it, so it surfaces as-is instead of as an engine failure.
+export class ExchangeRefusedError extends TransportableError {
+ constructor(code: ExchangeRefusalCode, message: string) {
+ super(code, message);
+ }
+
+ static deserialize(
+ code: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new ExchangeRefusedError(
+ code as ExchangeRefusalCode,
+ data.message,
+ );
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class SiteError extends TransportableError {
constructor(public errorCode: string) {
const errorExplanations: Record<string, string> = {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 36c264b3b..16f6ae239 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -60,6 +60,7 @@ import {
ScrapeRetryLimitError,
BrandingNotSupportedError,
XTwitterConfigurationError,
+ ExchangeRefusedError,
} from "./error";
import { ScrapeRetryTracker } from "./retryTracker";
import { executeTransformers } from "./transformers";
@@ -106,7 +107,10 @@ import { writeFile } from "node:fs/promises";
import path from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
-import type { ExchangeScrapeMetadata } from "../../lib/exchange";
+import {
+ type ExchangeScrapeMetadata,
+ ThirdPartyDataTermsRequiredError,
+} from "../../lib/exchange";
import {
checkUrl,
type ThreatCheckDedup,
@@ -167,6 +171,9 @@ export type Meta = {
abort: AbortManager;
featureFlags: Set<FeatureFlag>;
mock: MockState | null;
+ /** The Exchange provider whose access was checked when the exchange engine
+ * was chosen, so the engine asks the Exchange for that same provider. */
+ exchangeProviderId?: string;
/** Whether this scrape may OCR raster images: the request's parsers
* include `image` (the default; a parse upload of an image always counts)
* and the deployment has image OCR switched on with FirePDF configured.
@@ -1006,7 +1013,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error.error instanceof ProxySelectionError ||
error.error instanceof NoCachedDataError ||
error.error instanceof AgentIndexOnlyError ||
- error.error instanceof XTwitterConfigurationError
+ error.error instanceof XTwitterConfigurationError ||
+ error.error instanceof ExchangeRefusedError ||
+ error.error instanceof ThirdPartyDataTermsRequiredError
) {
throw error.error;
} else if (error.error instanceof LLMRefusalError) {
@@ -1155,7 +1164,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
blocks: engineResult.blocks,
rawHtml: engineResult.html,
rawBase64: engineResult.rawBase64,
- json: engineResult.json,
screenshot: engineResult.screenshot,
actions: engineResult.actions,
branding: engineResult.branding,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.test.ts b/apps/api/src/scraper/scrapeURL/transformers/index.test.ts
index 1e87844b0..989c451e1 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.test.ts
@@ -37,15 +37,20 @@ describe("executeTransformers", () => {
mockedPerformLLMExtract.mockClear();
});
- it("keeps native JSON and markdown without running LLM JSON extraction", async () => {
- const nativeJson = { id: "person-1", full_name: "Example Person" };
- const nativeMarkdown = "# Example Person";
+ it("runs JSON extraction on markdown an engine delivered natively", async () => {
+ mockedPerformLLMExtract.mockImplementationOnce(async (_meta, document) => ({
+ ...document,
+ json: { name: "Example Person" },
+ }));
const document = await executeTransformers(
{
url: "https://www.linkedin.com/in/example",
options: {
- formats: [{ type: "markdown" }, { type: "json" }],
+ formats: [
+ { type: "markdown" },
+ { type: "json", schema: { type: "object" } },
+ ],
onlyMainContent: false,
},
internalOptions: {},
@@ -53,41 +58,8 @@ describe("executeTransformers", () => {
} as any,
{
rawHtml:
- "<html><head><title>Ignored</title></head><body></body></html>",
- markdown: nativeMarkdown,
- json: nativeJson,
- metadata: {
- sourceURL: "https://www.linkedin.com/in/example",
- url: "https://www.linkedin.com/in/example",
- statusCode: 200,
- contentType: "text/markdown; charset=utf-8",
- },
- } as any,
- );
-
- expect(mockedPerformLLMExtract).not.toHaveBeenCalled();
- expect(document.markdown).toBe(nativeMarkdown);
- expect(document.json).toEqual(nativeJson);
- });
-
- it("maps native JSON to v1 extract compatibility field", async () => {
- const nativeJson = { id: "person-1", full_name: "Example Person" };
-
- const document = await executeTransformers(
- {
- url: "https://www.linkedin.com/in/example",
- options: {
- formats: [{ type: "markdown" }, { type: "json" }],
- onlyMainContent: false,
- },
- internalOptions: { v1OriginalFormat: "extract" },
- logger: logger(),
- } as any,
- {
- rawHtml:
- "<html><head><title>Ignored</title></head><body></body></html>",
+ "<html><head><title>Example</title></head><body><h1>Example Person</h1></body></html>",
markdown: "# Example Person",
- json: nativeJson,
metadata: {
sourceURL: "https://www.linkedin.com/in/example",
url: "https://www.linkedin.com/in/example",
@@ -97,8 +69,8 @@ describe("executeTransformers", () => {
} as any,
);
- expect(mockedPerformLLMExtract).not.toHaveBeenCalled();
- expect(document.extract).toEqual(nativeJson);
- expect(document.json).toEqual(nativeJson);
+ expect(mockedPerformLLMExtract).toHaveBeenCalledTimes(1);
+ expect(document.markdown).toBe("# Example Person");
+ expect(document.json).toEqual({ name: "Example Person" });
});
});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index e447aab2d..c899263c4 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -339,30 +339,6 @@ async function deriveBrandingFromActions(
return document;
}
-async function performLLMExtractUnlessNativeJson(
- meta: Meta,
- document: Document,
-): Promise<Document> {
- if (
- document.json !== undefined &&
- hasFormatOfType(meta.options.formats, "json")
- ) {
- if (
- meta.internalOptions.v1OriginalFormat === "extract" &&
- document.extract === undefined
- ) {
- document.extract = document.json;
- }
-
- meta.logger.debug(
- "Skipping LLM JSON extraction - document already has native JSON",
- );
- return document;
- }
-
- return performLLMExtract(meta, document);
-}
-
function coerceFieldsToFormats(meta: Meta, document: Document): Document {
const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");
const hasRawHtml = hasFormatOfType(meta.options.formats, "rawHtml");
@@ -653,7 +629,7 @@ const transformerStack: Transformer[] = [
fetchMenu,
...(useIndex ? [sendDocumentToIndex] : []),
...(useSearchIndex ? [sendDocumentToSearchIndex] : []), // Add to search index for real-time search
- performLLMExtractUnlessNativeJson,
+ performLLMExtract,
performDeterministicJson,
performSummary,
performQuery,
diff --git a/apps/api/src/services/alexandria/access.ts b/apps/api/src/services/alexandria/access.ts
index c4f8c50b4..ec317d82a 100644
--- a/apps/api/src/services/alexandria/access.ts
+++ b/apps/api/src/services/alexandria/access.ts
@@ -1,10 +1,15 @@
import { z } from "zod";
import { config } from "../../config";
import type { TeamFlags } from "../../controllers/v2/types";
-import { getThirdPartyDataTermsRequiredResponse } from "../../lib/exchange";
+import { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";
import { exchangeRequest } from "./client";
import { refusal, type ExchangeResponse, type ProviderCall } from "./contracts";
-import { acceptedProviders, type LedgerAcceptance } from "./terms";
+import {
+ acceptedAfter,
+ acceptedProviders,
+ matchesAcceptance,
+ type LedgerAcceptance,
+} from "./terms";
import { autumnService } from "../autumn/autumn.service";
import { HOBBY_RATE_LIMIT_MULTIPLIER } from "../rate-limiter";
@@ -29,24 +34,6 @@ const requirementsSchema = z.object({
),
});
-const timestampSchema = z.iso.datetime({ offset: true });
-
-function timestamp(value: unknown): number {
- const parsed = timestampSchema.safeParse(value);
- return parsed.success ? Date.parse(parsed.data) : NaN;
-}
-
-function matchesAcceptance(
- accepted: LedgerAcceptance | undefined,
- terms: { version: string; digest?: string },
-): boolean {
- return (
- terms.digest !== undefined &&
- accepted?.version === terms.version &&
- accepted.textHash === terms.digest
- );
-}
-
export async function authorizeProviders(
teamId: string,
calls: ProviderCall[],
@@ -92,18 +79,14 @@ export async function authorizeProviders(
if (revokedByOwner && item.required && item.terms && orgId !== null) {
ledger ??= await acceptedProviders(teamId, orgId);
const accepted = ledger.get(item.provider);
- const acceptedAt = timestamp(accepted?.acceptedAt);
- const disabledAt = timestamp(access.disabledAt);
if (
matchesAcceptance(accepted, item.terms) &&
- Number.isFinite(acceptedAt) &&
- Number.isFinite(disabledAt) &&
- acceptedAt > disabledAt
+ acceptedAfter(accepted, access.disabledAt)
)
continue;
return {
status: 403,
- body: getThirdPartyDataTermsRequiredResponse(item.terms),
+ body: new ThirdPartyDataTermsRequiredError(item.terms).response(),
};
}
return refusal(
@@ -124,7 +107,7 @@ export async function authorizeProviders(
}
return {
status: 403,
- body: getThirdPartyDataTermsRequiredResponse(item.terms),
+ body: new ThirdPartyDataTermsRequiredError(item.terms).response(),
};
}
diff --git a/apps/api/src/services/alexandria/terms.ts b/apps/api/src/services/alexandria/terms.ts
index deff0a7e2..8f042bb7c 100644
--- a/apps/api/src/services/alexandria/terms.ts
+++ b/apps/api/src/services/alexandria/terms.ts
@@ -57,14 +57,48 @@ export type LedgerAcceptance = {
acceptedAt?: string | null;
};
+const timestampSchema = z.iso.datetime({ offset: true });
+
+function timestamp(value: unknown): number {
+ const parsed = timestampSchema.safeParse(value);
+ return parsed.success ? Date.parse(parsed.data) : NaN;
+}
+
+export function matchesAcceptance(
+ accepted: LedgerAcceptance | undefined,
+ terms: { version: string; digest?: string },
+): boolean {
+ return (
+ terms.digest !== undefined &&
+ accepted?.version === terms.version &&
+ accepted.textHash === terms.digest
+ );
+}
+
+// Whether the acceptance was recorded after `since`, e.g. an admin revocation
+// it has to lift. Unparseable timestamps never qualify.
+export function acceptedAfter(
+ accepted: LedgerAcceptance | undefined,
+ since: unknown,
+): boolean {
+ const acceptedAt = timestamp(accepted?.acceptedAt);
+ const sinceAt = timestamp(since);
+ return (
+ Number.isFinite(acceptedAt) &&
+ Number.isFinite(sinceAt) &&
+ acceptedAt > sinceAt
+ );
+}
+
export async function acceptedProviders(
teamId: string,
orgId: string,
+ timeoutMs = TIMEOUT_MS,
): Promise<Map<string, LedgerAcceptance>> {
const response = await exchangeRequest({
teamId,
path: `/v1/provider-terms/status?organizationId=${encodeURIComponent(orgId)}`,
- timeoutMs: TIMEOUT_MS,
+ timeoutMs,
}).catch(() => undefined);
const parsed =
response?.status === 200
@@ -84,6 +118,45 @@ export async function acceptedProviders(
return accepted;
}
+/**
+ * Whether the organization's ledger on the Exchange holds an acceptance of the
+ * provider's current terms, recorded after `revocation.disabledAt` when an
+ * admin revocation has to be lifted. The API's accept route records
+ * acceptance only there. Any failure reads as not accepted.
+ */
+export async function hasLedgerAcceptance(input: {
+ teamId: string;
+ orgId: string;
+ provider: string;
+ revocation?: { disabledAt: unknown };
+ timeoutMs: number;
+}): Promise<boolean> {
+ const [requirements, ledger] = await Promise.all([
+ exchangeRequest({
+ teamId: input.teamId,
+ path: "/v1/provider-terms/requirements",
+ body: { providers: [input.provider] },
+ timeoutMs: input.timeoutMs,
+ }).catch(() => undefined),
+ acceptedProviders(input.teamId, input.orgId, input.timeoutMs),
+ ]);
+ const parsed =
+ requirements?.status === 200
+ ? requirementsSchema.safeParse(requirements.body)
+ : undefined;
+ const terms = parsed?.success
+ ? parsed.data.providers.find(item => item.provider === input.provider)
+ ?.terms
+ : undefined;
+ if (!terms) return false;
+ const accepted = ledger.get(input.provider);
+ return (
+ matchesAcceptance(accepted, terms) &&
+ (input.revocation === undefined ||
+ acceptedAfter(accepted, input.revocation.disabledAt))
+ );
+}
+
export async function acceptProviderTerms(input: {
teamId: string;
orgId: string;
diff --git a/apps/api/src/services/index.test.ts b/apps/api/src/services/index.test.ts
deleted file mode 100644
index ba7507e7d..000000000
--- a/apps/api/src/services/index.test.ts
+++ /dev/null
@@ -1,105 +0,0 @@
-import { vi } from "vitest";
-
-const { file, bucket, getSavedPayload, setSavedPayload } = vi.hoisted(() => {
- let savedPayload = "";
- const getSavedPayload = () => savedPayload;
- const setSavedPayload = (value: string) => {
- savedPayload = value;
- };
- const file = vi.fn(() => ({
- save: vi.fn(async (payload: string) => {
- savedPayload = payload;
- }),
- download: vi.fn(async () => [Buffer.from(savedPayload)]),
- }));
- const bucket = vi.fn(() => ({ file }));
-
- return { file, bucket, getSavedPayload, setSavedPayload };
-});
-
-vi.mock("../config", () => ({
- config: {
- GCS_INDEX_BUCKET_NAME: "index-bucket",
- GCS_MEDIA_BUCKET_NAME: "media-bucket",
- },
-}));
-
-vi.mock("../lib/gcs-jobs", () => ({
- storage: { bucket },
-}));
-
-vi.mock("../lib/otel-tracer", () => ({
- withSpan: vi.fn(async (_name, fn) => fn({})),
- setSpanAttributes: vi.fn(),
-}));
-
-vi.mock("../db/connection", () => ({
- dbIndex: {},
-}));
-
-vi.mock("../db/rpc", () => ({
- insertOmceJobIfNeeded: vi.fn(),
- queryIndexAtSplitLevel: vi.fn(),
- queryIndexAtDomainSplitLevel: vi.fn(),
- queryOmceSignatures: vi.fn(),
- queryEngpickerVerdict: vi.fn(),
- queryIndexAtSplitLevelWithMeta: vi.fn(),
- queryIndexAtDomainSplitLevelWithMeta: vi.fn(),
- queryDomainPriority: vi.fn(),
-}));
-
-vi.mock("./redis", () => ({
- redisEvictConnection: {},
-}));
-
-vi.mock("./index-cache", () => ({
- deriveIndexVariantKey: vi.fn(),
- upsertCachedIndexEntries: vi.fn(),
- useIndexCache: false,
-}));
-
-vi.mock("../db/schema", () => ({}));
-
-import { getIndexFromGCS, saveIndexToGCS } from "./index";
-
-describe("index GCS documents", () => {
- beforeEach(() => {
- vi.clearAllMocks();
- setSavedPayload("");
- });
-
- it("persists native JSON payloads with cached index documents", async () => {
- await saveIndexToGCS("idx-1", {
- url: "https://www.example.com/profile",
- html: "# Example",
- json: { id: "person-1", full_name: "Example Person" },
- statusCode: 200,
- contentType: "text/markdown; charset=utf-8",
- proxyUsed: "basic",
- });
-
- expect(file).toHaveBeenCalledWith("idx-1.json");
- expect(JSON.parse(getSavedPayload()).json).toEqual({
- id: "person-1",
- full_name: "Example Person",
- });
- });
-
- it("reads native JSON payloads from cached index documents", async () => {
- setSavedPayload(
- JSON.stringify({
- url: "https://www.example.com/profile",
- html: "# Example",
- json: { id: "person-1", full_name: "Example Person" },
- statusCode: 200,
- }),
- );
-
- const doc = await getIndexFromGCS("idx-1.json");
-
- expect(doc?.json).toEqual({
- id: "person-1",
- full_name: "Example Person",
- });
- });
-});
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index ceef42295..3a047e6ee 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -118,7 +118,6 @@ export async function saveIndexToGCS(
doc: {
url: string;
html: string;
- json?: unknown;
statusCode: number;
error?: string;
screenshot?: string;
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
index 9f4bbc47c..8b0564bf1 100644
--- a/apps/js-sdk/firecrawl/src/v1/index.ts
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -2,6 +2,7 @@ import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from
import * as zt from "zod";
import { zodSchemaToJsonSchema } from "../utils/zodSchemaToJson";
import { TypedEventTarget } from "typescript-event-target";
+import type { RequiresAction } from "../v2/types";
/**
* Configuration interface for FirecrawlApp.
@@ -439,6 +440,8 @@ export interface CrawlErrorsResponse {
url: string,
code?: string,
error: string,
+ /** Present when the page needs provider terms accepted first. */
+ requiresAction?: RequiresAction,
}[];
/**
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
index 90bfd4665..9683cffae 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
@@ -152,7 +152,7 @@ export async function getBatchScrapeErrors(
try {
const res = await http.get<{
success?: boolean;
- data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] };
+ data?: CrawlErrorsResponse;
}>(`/v2/batch/scrape/${jobId}/errors`);
if (res.status !== 200) throwForBadResponse(res, "get batch scrape errors");
const payload = res.data?.data ?? (res.data as any);
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
index 7a46c5aad..834444c13 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
@@ -162,7 +162,7 @@ export async function crawl(http: HttpClient, request: CrawlRequest, pollInterva
export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {
try {
- const res = await http.get<{ success?: boolean; data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] } }>(`/v2/crawl/${crawlId}/errors`);
+ const res = await http.get<{ success?: boolean; data?: CrawlErrorsResponse }>(`/v2/crawl/${crawlId}/errors`);
if (res.status !== 200) throwForBadResponse(res, "get crawl errors");
const payload = res.data?.data ?? (res.data as any);
return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index b048701ed..fb0b6f8fc 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -1955,6 +1955,8 @@ export interface CrawlErrorsResponse {
url: string;
code?: string;
error: string;
+ /** Present when the page needs provider terms accepted first. */
+ requiresAction?: RequiresAction;
}[];
robotsBlocked: string[];
}
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_errors.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_errors.py
new file mode 100644
index 000000000..4951d4042
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_errors.py
@@ -0,0 +1,61 @@
+from unittest.mock import Mock
+
+from firecrawl.v1.client import V1CrawlErrorsResponse
+from firecrawl.v2.methods.batch import get_batch_scrape_errors
+from firecrawl.v2.methods.crawl import get_crawl_errors
+
+REQUIRES_ACTION = {
+ "type": "accept_terms",
+ "terms": "acme",
+ "version": "2026-01-01",
+ "url": "https://www.firecrawl.dev/app/alexandria/acme",
+}
+
+ERRORS = {
+ "errors": [
+ {
+ "id": "job-terms",
+ "url": "https://profiles.example/person/example-person",
+ "code": "THIRD_PARTY_DATA_TERMS_REQUIRED",
+ "error": "An organization admin must accept the acme provider's terms.",
+ "requiresAction": REQUIRES_ACTION,
+ },
+ {
+ "id": "job-site",
+ "url": "https://down.example/",
+ "code": "SCRAPE_SITE_ERROR",
+ "error": "The connection was reset by the peer.",
+ },
+ ],
+ "robotsBlocked": [],
+}
+
+
+def _client():
+ response = Mock()
+ response.ok = True
+ response.json.return_value = ERRORS
+ client = Mock()
+ client.get.return_value = response
+ return client
+
+
+def test_crawl_errors_carry_requires_action():
+ result = get_crawl_errors(_client(), "crawl-1")
+
+ assert result.errors[0].requires_action == REQUIRES_ACTION
+ assert result.errors[1].requires_action is None
+
+
+def test_batch_scrape_errors_carry_requires_action():
+ result = get_batch_scrape_errors(_client(), "batch-1")
+
+ assert result.errors[0].requires_action == REQUIRES_ACTION
+ assert result.errors[1].requires_action is None
+
+
+def test_v1_crawl_errors_carry_requires_action():
+ result = V1CrawlErrorsResponse(**ERRORS)
+
+ assert result.errors[0].requiresAction == REQUIRES_ACTION
+ assert result.errors[1].requiresAction is None
diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.py
index c3c1b884b..b20a32096 100644
--- a/apps/python-sdk/firecrawl/v1/client.py
+++ b/apps/python-sdk/firecrawl/v1/client.py
@@ -295,6 +295,9 @@ class V1CrawlError(pydantic.BaseModel):
url: str
code: Optional[str] = None
error: str
+ # Set when the page needs provider terms accepted first:
+ # {"type": "accept_terms", "terms", "version", "url"}.
+ requiresAction: Optional[Dict[str, Any]] = None
class V1CrawlErrorsResponse(pydantic.BaseModel):
"""Response from crawl/batch scrape error monitoring."""
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index fda30a374..ecf2f4dab 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -2544,11 +2544,18 @@ class JobStatus(BaseModel):
class CrawlError(BaseModel):
"""A crawl error."""
+ model_config = {"populate_by_name": True}
+
id: str
timestamp: Optional[datetime] = None
url: str
code: Optional[str] = None
error: str
+ # Set when the page needs provider terms accepted first:
+ # {"type": "accept_terms", "terms", "version", "url"}.
+ requires_action: Optional[Dict[str, Any]] = Field(
+ default=None, alias="requiresAction"
+ )
class CrawlErrorsResponse(BaseModel):