Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): return 4xx for Exchange not_found and invalid_exchange_url refusals (#4998)

Gergő Móriczecd7934e0478
diff --git a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts
index 531c99840..f0712b0b9 100644
--- a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts
+++ b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts
@@ -258,6 +258,68 @@ describe("Exchange routing", () => {
);
});
+ it("serves the Exchange's markdown for a supported URL", async () => {
+ mocks.robustFetch.mockResolvedValue({
+ success: true,
+ accessEventId: "access-1",
+ creditsCost: 12,
+ data: {
+ url: BLOCKED_URL,
+ title: "Example Person",
+ markdown: "# Example Person",
+ source: { provider: "acme" },
+ },
+ });
+
+ const result = await scrapeURLWithExchange(
+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),
+ );
+
+ expect(result).toMatchObject({
+ url: BLOCKED_URL,
+ markdown: "# Example Person",
+ statusCode: 200,
+ exchange: {
+ handled: true,
+ creditsCost: 12,
+ accessEventId: "access-1",
+ integrationId: "acme",
+ },
+ });
+ });
+
+ it("reports a provider's missing record as THIRD_PARTY_DATA_NOT_FOUND", async () => {
+ mocks.robustFetch.mockResolvedValue(
+ failure("not_found", "No matching profile was found."),
+ );
+
+ const error = await scrapeURLWithExchange(
+ buildStubMeta(BLOCKED_URL, ACCEPTED_FLAGS),
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(scrapeErrors.ExchangeRefusedError);
+ expect(error.code).toBe("THIRD_PARTY_DATA_NOT_FOUND");
+ expect(error.message).toBe("No matching profile was found.");
+ });
+
+ it("reports a URL the provider does not serve as THIRD_PARTY_DATA_UNSUPPORTED_URL", async () => {
+ mocks.robustFetch.mockResolvedValue(
+ failure(
+ "invalid_exchange_url",
+ "Expected a canonical Exchange URL or supported source URL.",
+ ),
+ );
+
+ const error = await scrapeURLWithExchange(
+ buildStubMeta(`${BLOCKED_URL}/details/experience/`, ACCEPTED_FLAGS),
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(scrapeErrors.ExchangeRefusedError);
+ expect(error.code).toBe("THIRD_PARTY_DATA_UNSUPPORTED_URL");
+ expect(error.message).not.toContain("canonical Exchange URL");
+ expect(error.message).toContain("record's own page");
+ });
+
it("reports a provider the team lacks as THIRD_PARTY_DATA_NOT_ENABLED", async () => {
mocks.robustFetch.mockResolvedValue(
failure(
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 0f0dc794b..fe9805c34 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -431,6 +431,14 @@ async function scrapeControllerInner(
});
}
+ if (e.code === "THIRD_PARTY_DATA_UNSUPPORTED_URL") {
+ return res.status(400).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
return res.status(timeoutErr ? 408 : 500).json({
success: false,
code: e.code,
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index f01f5bd2e..7b8dddd32 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -625,6 +625,17 @@ export async function scrapeController(
});
}
+ if (e.code === "THIRD_PARTY_DATA_UNSUPPORTED_URL") {
+ setSpanAttributes(span, {
+ "scrape.status_code": 400,
+ });
+ return res.status(400).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
const statusCode = timeoutErr ? 408 : 500;
setSpanAttributes(span, {
"scrape.status_code": statusCode,
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 556f58e09..6d7d01ebf 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -51,6 +51,7 @@ import { ThirdPartyDataTermsRequiredError } from "./exchange";
const errorMap: Record<ErrorCodes, any> = {
THIRD_PARTY_DATA_TERMS_REQUIRED: ThirdPartyDataTermsRequiredError,
THIRD_PARTY_DATA_NOT_FOUND: ExchangeRefusedError,
+ THIRD_PARTY_DATA_UNSUPPORTED_URL: ExchangeRefusedError,
THIRD_PARTY_DATA_NOT_ENABLED: ExchangeRefusedError,
THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED: ExchangeRefusedError,
SAFE_MODE_BLOCKED: null,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index b32d40f53..4b4b93e7b 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -3,6 +3,7 @@ import { UNSUPPORTED_SITE_MESSAGE } from "./strings";
export type ErrorCodes =
| "THIRD_PARTY_DATA_TERMS_REQUIRED"
| "THIRD_PARTY_DATA_NOT_FOUND"
+ | "THIRD_PARTY_DATA_UNSUPPORTED_URL"
| "THIRD_PARTY_DATA_NOT_ENABLED"
| "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED"
| "SCRAPE_TIMEOUT"
diff --git a/apps/api/src/scraper/scrapeURL/engines/exchange.ts b/apps/api/src/scraper/scrapeURL/engines/exchange.ts
index 60239dbfc..1aa0f58c4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/exchange.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/exchange.ts
@@ -62,6 +62,9 @@ const EXCHANGE_REFUSALS = new Map<
code: ConstructorParameters<typeof ExchangeRefusedError>[0];
message: string;
enrichmentSettings?: true;
+ // Set when the Exchange's message describes its own URL contract, which
+ // means nothing to a scrape caller.
+ ignoreExchangeMessage?: true;
}
>([
[
@@ -72,6 +75,23 @@ const EXCHANGE_REFUSALS = new Map<
"The third-party data provider for this URL has no record for it.",
},
],
+ [
+ "not_found",
+ {
+ code: "THIRD_PARTY_DATA_NOT_FOUND",
+ message:
+ "The third-party data provider for this URL has no record for it.",
+ },
+ ],
+ [
+ "invalid_exchange_url",
+ {
+ code: "THIRD_PARTY_DATA_UNSUPPORTED_URL",
+ message:
+ "The third-party data provider for this site does not serve this URL. Request a record's own page, such as a profile page itself, rather than a sub-page or listing under it.",
+ ignoreExchangeMessage: true,
+ },
+ ],
[
"provider_not_enabled",
{
@@ -207,7 +227,9 @@ export async function scrapeURLWithExchange(
}
const refusal = EXCHANGE_REFUSALS.get(response.error?.code ?? "");
if (refusal !== undefined) {
- const message = response.error?.message || refusal.message;
+ const message = refusal.ignoreExchangeMessage
+ ? refusal.message
+ : response.error?.message || refusal.message;
throw new ExchangeRefusedError(
refusal.code,
refusal.enrichmentSettings
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 59bc1beec..cfe3894ec 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -123,12 +123,14 @@ export class SSLError extends TransportableError {
type ExchangeRefusalCode =
| "THIRD_PARTY_DATA_NOT_FOUND"
+ | "THIRD_PARTY_DATA_UNSUPPORTED_URL"
| "THIRD_PARTY_DATA_NOT_ENABLED"
| "THIRD_PARTY_DATA_ENRICHMENT_NOT_ENABLED";
// A definitive answer from the Exchange about this URL: the provider holds no
-// record for it, or the team is not entitled to the provider. Another attempt
-// cannot change it, so it surfaces as-is instead of as an engine failure.
+// record for it, does not serve it, or the team is not entitled to the
+// provider. Another attempt cannot change it, so it surfaces as-is instead of
+// as an engine failure.
export class ExchangeRefusedError extends TransportableError {
constructor(code: ExchangeRefusalCode, message: string) {
super(code, message);