Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): return 4xx for Exchange not_found and invalid_exchange_url refusals (#4998)
Gergő Móriczecd7934e0478
diff --git a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts b/apps/api/src/__tests__/snips/v2/exchange-routing.test.tsindex 531c99840..f0712b0b9 100644--- a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts+++ b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts@@ -258,6 +258,68 @@ describe("Exchange routing", () => {);});+ it("serves the Exchange's markdown for a supported URL", async () => {+ mocks.robustFetch.mockResolvedValue({+ success: true,+ accessEventId: "access-1",+ creditsCost: 12,+ data: {+ url: BLOCKED_URL,+ title: "Example Person",+ markdown: "# Example Person",+ source: { provider: "acme" },+ },+ });++ const result = await scrapeURLWithExchange(+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),+ );++ expect(result).toMatchObject({+ url: BLOCKED_URL,+ markdown: "# Example Person",+ statusCode: 200,+ exchange: {+ handled: true,+ creditsCost: 12,+ accessEventId: "access-1",+ integrationId: "acme",+ },+ });+ });++ it("reports a provider's missing record as THIRD_PARTY_DATA_NOT_FOUND", async () => {+ mocks.robustFetch.mockResolvedValue(+ failure("not_found", "No matching profile was found."),+ );++ const error = await scrapeURLWithExchange(+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),+ ).catch(e => e);++ expect(error).toBeInstanceOf(scrapeErrors.ExchangeRefusedError);+ expect(error.code).toBe("THIRD_PARTY_DATA_NOT_FOUND");+ expect(error.message).toBe("No matching profile was found.");+ });++ it("reports a URL the provider does not serve as THIRD_PARTY_DATA_UNSUPPORTED_URL", async () => {+ mocks.robustFetch.mockResolvedValue(+ failure(+ "invalid_exchange_url",+ "Expected a canonical Exchange URL or supported source URL.",+ ),+ );++ const error = await scrapeURLWithExchange(+ buildStubMeta(`${BLOCKED_URL}/details/experience/`, ACCEPTED_FLAGS),+ ).catch(e => e);++ expect(error).toBeInstanceOf(scrapeErrors.ExchangeRefusedError);+ expect(error.code).toBe("THIRD_PARTY_DATA_UNSUPPORTED_URL");+ expect(error.message).not.toContain("canonical Exchange URL");+ expect(error.message).toContain("record's own page");+ });+it("reports a provider the team lacks as THIRD_PARTY_DATA_NOT_ENABLED", async () => {mocks.robustFetch.mockResolvedValue(failure(diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 0f0dc794b..fe9805c34 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -431,6 +431,14 @@ async function scrapeControllerInner(});}+ if (e.code === "THIRD_PARTY_DATA_UNSUPPORTED_URL") {+ return res.status(400).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+return res.status(timeoutErr ? 408 : 500).json({success: false,code: e.code,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex f01f5bd2e..7b8dddd32 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -625,6 +625,17 @@ export async function scrapeController(});}+ if (e.code === "THIRD_PARTY_DATA_UNSUPPORTED_URL") {+ setSpanAttributes(span, {+ "scrape.status_code": 400,+ });+ return res.status(400).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+const statusCode = timeoutErr ? 408 : 500;setSpanAttributes(span, {"scrape.status_code": statusCode,diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 556f58e09..6d7d01ebf 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -51,6 +51,7 @@ import { ThirdPartyDataTermsRequiredError } from "./exchange";const errorMap: Record<ErrorCodes, any> = {THIRD_PARTY_DATA_TERMS_REQUIRED: ThirdPartyDataTermsRequiredError,THIRD_PARTY_DATA_NOT_FOUND: ExchangeRefusedError,+ THIRD_PARTY_DATA_UNSUPPORTED_URL: ExchangeRefusedError,THIRD_PARTY_DATA_NOT_ENABLED: ExchangeRefusedError,THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED: ExchangeRefusedError,SAFE_MODE_BLOCKED: null,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex b32d40f53..4b4b93e7b 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -3,6 +3,7 @@ import { UNSUPPORTED_SITE_MESSAGE } from "./strings";export type ErrorCodes =| "THIRD_PARTY_DATA_TERMS_REQUIRED"| "THIRD_PARTY_DATA_NOT_FOUND"+ | "THIRD_PARTY_DATA_UNSUPPORTED_URL"| "THIRD_PARTY_DATA_NOT_ENABLED"| "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED"| "SCRAPE_TIMEOUT"diff --git a/apps/api/src/scraper/scrapeURL/engines/exchange.ts b/apps/api/src/scraper/scrapeURL/engines/exchange.tsindex 60239dbfc..1aa0f58c4 100644--- a/apps/api/src/scraper/scrapeURL/engines/exchange.ts+++ b/apps/api/src/scraper/scrapeURL/engines/exchange.ts@@ -62,6 +62,9 @@ const EXCHANGE_REFUSALS = new Map<code: ConstructorParameters<typeof ExchangeRefusedError>[0];message: string;enrichmentSettings?: true;+ // Set when the Exchange's message describes its own URL contract, which+ // means nothing to a scrape caller.+ ignoreExchangeMessage?: true;}>([[@@ -72,6 +75,23 @@ const EXCHANGE_REFUSALS = new Map<"The third-party data provider for this URL has no record for it.",},],+ [+ "not_found",+ {+ code: "THIRD_PARTY_DATA_NOT_FOUND",+ message:+ "The third-party data provider for this URL has no record for it.",+ },+ ],+ [+ "invalid_exchange_url",+ {+ code: "THIRD_PARTY_DATA_UNSUPPORTED_URL",+ message:+ "The third-party data provider for this site does not serve this URL. Request a record's own page, such as a profile page itself, rather than a sub-page or listing under it.",+ ignoreExchangeMessage: true,+ },+ ],["provider_not_enabled",{@@ -207,7 +227,9 @@ export async function scrapeURLWithExchange(}const refusal = EXCHANGE_REFUSALS.get(response.error?.code ?? "");if (refusal !== undefined) {- const message = response.error?.message || refusal.message;+ const message = refusal.ignoreExchangeMessage+ ? refusal.message+ : response.error?.message || refusal.message;throw new ExchangeRefusedError(refusal.code,refusal.enrichmentSettingsdiff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 59bc1beec..cfe3894ec 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -123,12 +123,14 @@ export class SSLError extends TransportableError {type ExchangeRefusalCode =| "THIRD_PARTY_DATA_NOT_FOUND"+ | "THIRD_PARTY_DATA_UNSUPPORTED_URL"| "THIRD_PARTY_DATA_NOT_ENABLED"| "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED";// A definitive answer from the Exchange about this URL: the provider holds no-// record for it, or the team is not entitled to the provider. Another attempt-// cannot change it, so it surfaces as-is instead of as an engine failure.+// record for it, does not serve it, or the team is not entitled to the+// provider. Another attempt cannot change it, so it surfaces as-is instead of+// as an engine failure.export class ExchangeRefusedError extends TransportableError {constructor(code: ExchangeRefusalCode, message: string) {super(code, message);