Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): never route Exchange-originated scrapes back to the Exchange (#5563)
Gergő Móricz5ca5e53bde53
diff --git a/apps/api/src/lib/exchange-request.ts b/apps/api/src/lib/exchange-request.tsindex 245f8c17e..57e9f6c4f 100644--- a/apps/api/src/lib/exchange-request.ts+++ b/apps/api/src/lib/exchange-request.ts@@ -28,6 +28,7 @@ export function getExchangeAccessForRequestBody(input: {teamId: input.teamId,orgId: input.orgId,blocked: input.blocked,+ origin: typeof body.origin === "string" ? body.origin : null,formats: scrapeOptions.formats,actions: scrapeOptions.actions,headers: scrapeOptions.headers,diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.tsindex 6781ede9a..8557bd6e5 100644--- a/apps/api/src/lib/exchange.test.ts+++ b/apps/api/src/lib/exchange.test.ts@@ -535,6 +535,29 @@ describe("Exchange routing", () => {});});+ it("never routes Exchange-originated requests back to the Exchange", async () => {+ const request = {+ url: "https://facts.example/records/1",+ formats: [{ type: "markdown" }],+ flags: { professionalProfileCompanyDataBeta: true },+ };++ await expect(+ canUseExchangeForRequest({ ...request, origin: "api" }),+ ).resolves.toBe(true);+ await expect(+ getExchangeAccessForRequest({ ...request, origin: "exchange" }),+ ).resolves.toEqual({ allowed: false, termsRequired: false });+ // A blocked URL stays refused rather than offered to the Exchange.+ await expect(+ getExchangeAccessForRequest({+ ...request,+ origin: "exchange",+ blocked: true,+ }),+ ).resolves.toEqual({ allowed: false, termsRequired: false });+ });+it("does not route unless the Exchange is configured", async () => {config.FIRE_EXCHANGE_URL = undefined;diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.tsindex a9bc26dc3..04467a465 100644--- a/apps/api/src/lib/exchange.ts+++ b/apps/api/src/lib/exchange.ts@@ -28,6 +28,7 @@ type RouteInput = {orgId?: string | null;/** The URL is on the team's blocklist, so the Exchange is the only way to serve it. */blocked?: boolean;+ origin?: string | null;formats?: FormatObject[] | unknown[];actions?: unknown[];headers?: Record<string, unknown>;@@ -541,6 +542,11 @@ async function getProviderAccessDecision(}function isExchangeEligibleRequest(input: RouteInput): boolean {+ // The Exchange scrapes through Firecrawl; routing those back would loop.+ if (input.origin === "exchange") {+ return false;+ }+// Blocked URLs go to the Exchange for every team, since nothing else may// serve them; elsewhere it replaces a normal scrape only for the beta.if (input.flags?.[EXCHANGE_BETA_FLAG] !== true && input.blocked !== true) {diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex c52d15f8f..a0ec66f46 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -28,6 +28,7 @@ export async function startWebScraperPipeline({internalOptions: {crawlId: job.data.crawl_id,teamId: job.data.team_id,+ origin: job.data.origin,...job.data.internalOptions,// In-flight jobs enqueued before orgId existed may omit it.orgId: job.data.internalOptions?.orgId ?? null,diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 6ec6b3aa4..88815b510 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -706,6 +706,7 @@ export async function buildFallbackList(meta: Meta): Promise<teamId: meta.internalOptions.teamId ?? null,orgId: meta.internalOptions.orgId ?? null,blocked,+ origin: meta.internalOptions.origin ?? null,formats: meta.options.formats,actions: meta.options.actions,headers: meta.options.headers,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex e57ae9a35..b3cfa89c5 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -607,6 +607,7 @@ async function buildMetaObject(export type InternalOptions = {teamId: string;crawlId?: string;+ origin?: string;priority?: number; // Passed along to fire-engineforceEngine?: Engine | Engine[];