Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): never route Exchange-originated scrapes back to the Exchange (#5563)

Gergő Móricz5ca5e53bde53
diff --git a/apps/api/src/lib/exchange-request.ts b/apps/api/src/lib/exchange-request.ts
index 245f8c17e..57e9f6c4f 100644
--- a/apps/api/src/lib/exchange-request.ts
+++ b/apps/api/src/lib/exchange-request.ts
@@ -28,6 +28,7 @@ export function getExchangeAccessForRequestBody(input: {
teamId: input.teamId,
orgId: input.orgId,
blocked: input.blocked,
+ origin: typeof body.origin === "string" ? body.origin : null,
formats: scrapeOptions.formats,
actions: scrapeOptions.actions,
headers: scrapeOptions.headers,
diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.ts
index 6781ede9a..8557bd6e5 100644
--- a/apps/api/src/lib/exchange.test.ts
+++ b/apps/api/src/lib/exchange.test.ts
@@ -535,6 +535,29 @@ describe("Exchange routing", () => {
});
});
+ it("never routes Exchange-originated requests back to the Exchange", async () => {
+ const request = {
+ url: "https://facts.example/records/1",
+ formats: [{ type: "markdown" }],
+ flags: { professionalProfileCompanyDataBeta: true },
+ };
+
+ await expect(
+ canUseExchangeForRequest({ ...request, origin: "api" }),
+ ).resolves.toBe(true);
+ await expect(
+ getExchangeAccessForRequest({ ...request, origin: "exchange" }),
+ ).resolves.toEqual({ allowed: false, termsRequired: false });
+ // A blocked URL stays refused rather than offered to the Exchange.
+ await expect(
+ getExchangeAccessForRequest({
+ ...request,
+ origin: "exchange",
+ blocked: true,
+ }),
+ ).resolves.toEqual({ allowed: false, termsRequired: false });
+ });
+
it("does not route unless the Exchange is configured", async () => {
config.FIRE_EXCHANGE_URL = undefined;
diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.ts
index a9bc26dc3..04467a465 100644
--- a/apps/api/src/lib/exchange.ts
+++ b/apps/api/src/lib/exchange.ts
@@ -28,6 +28,7 @@ type RouteInput = {
orgId?: string | null;
/** The URL is on the team's blocklist, so the Exchange is the only way to serve it. */
blocked?: boolean;
+ origin?: string | null;
formats?: FormatObject[] | unknown[];
actions?: unknown[];
headers?: Record<string, unknown>;
@@ -541,6 +542,11 @@ async function getProviderAccessDecision(
}
function isExchangeEligibleRequest(input: RouteInput): boolean {
+ // The Exchange scrapes through Firecrawl; routing those back would loop.
+ if (input.origin === "exchange") {
+ return false;
+ }
+
// Blocked URLs go to the Exchange for every team, since nothing else may
// serve them; elsewhere it replaces a normal scrape only for the beta.
if (input.flags?.[EXCHANGE_BETA_FLAG] !== true && input.blocked !== true) {
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index c52d15f8f..a0ec66f46 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -28,6 +28,7 @@ export async function startWebScraperPipeline({
internalOptions: {
crawlId: job.data.crawl_id,
teamId: job.data.team_id,
+ origin: job.data.origin,
...job.data.internalOptions,
// In-flight jobs enqueued before orgId existed may omit it.
orgId: job.data.internalOptions?.orgId ?? null,
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 6ec6b3aa4..88815b510 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -706,6 +706,7 @@ export async function buildFallbackList(meta: Meta): Promise<
teamId: meta.internalOptions.teamId ?? null,
orgId: meta.internalOptions.orgId ?? null,
blocked,
+ origin: meta.internalOptions.origin ?? null,
formats: meta.options.formats,
actions: meta.options.actions,
headers: meta.options.headers,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index e57ae9a35..b3cfa89c5 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -607,6 +607,7 @@ async function buildMetaObject(
export type InternalOptions = {
teamId: string;
crawlId?: string;
+ origin?: string;
priority?: number; // Passed along to fire-engine
forceEngine?: Engine | Engine[];