Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): report the serving provider as metadata.provider on Exchange scrapes (#5000)

Gergő Móricza2833245bee7
diff --git a/apps/api/src/__tests__/snips/v2/exchange-provider-metadata.test.ts b/apps/api/src/__tests__/snips/v2/exchange-provider-metadata.test.ts
new file mode 100644
index 000000000..e0b09f1c8
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/exchange-provider-metadata.test.ts
@@ -0,0 +1,238 @@
+import { randomUUID } from "node:crypto";
+import { config } from "../../../config";
+import { scrapeOptions } from "../../../controllers/v2/types";
+import { CostTracking } from "../../../lib/cost-tracking";
+import { scrapeURL } from "../../../scraper/scrapeURL";
+import { scrapeURLWithExchange } from "../../../scraper/scrapeURL/engines/exchange";
+import { ALLOW_TEST_SUITE_WEBSITE, itIf, TEST_SUITE_WEBSITE } from "../lib";
+import { scrapeTimeout } from "./lib";
+
+const mocks = vi.hoisted(() => ({ robustFetch: vi.fn() }));
+
+vi.mock("../../../scraper/scrapeURL/lib/fetch", () => ({
+ robustFetch: mocks.robustFetch,
+}));
+
+const PROFILE_URL = "https://profiles.example/in/example-person";
+
+const WATERFALL_RESPONSE = {
+ success: true,
+ accessEventId: "access-1",
+ creditsCost: 30,
+ data: {
+ url: PROFILE_URL,
+ title: "Example Person",
+ markdown: "# Example Person",
+ source: {
+ provider: "firecrawl-enrich",
+ recordId: PROFILE_URL,
+ recordType: "professional_profile",
+ },
+ metadata: {
+ canonicalUrl: PROFILE_URL,
+ targetKind: "person",
+ enrichment: {
+ status: "matched",
+ preferenceVersion: 1,
+ source: { provider: "globex", capability: "person-profile" },
+ steps: [
+ {
+ provider: "acme",
+ capability: "person-profile",
+ requestId: "step-1",
+ status: "not_found",
+ creditsCost: 10,
+ },
+ {
+ provider: "globex",
+ capability: "person-profile",
+ requestId: "step-2",
+ status: "matched",
+ creditsCost: 20,
+ },
+ ],
+ providerCredits: 30,
+ },
+ },
+ },
+};
+
+const WATERFALL_PROVIDER = {
+ id: "globex",
+ creditsCost: 30,
+ steps: [
+ { provider: "acme", status: "not_found", creditsCost: 10 },
+ { provider: "globex", status: "matched", creditsCost: 20 },
+ ],
+};
+
+// Parses with the engine's own schema, as the real robustFetch does.
+function exchangeResponds(response: unknown) {
+ mocks.robustFetch.mockImplementation(async ({ schema }) =>
+ schema.parse(response),
+ );
+}
+
+function directResponse(metadata?: unknown) {
+ return {
+ success: true,
+ accessEventId: "access-2",
+ creditsCost: 12,
+ data: {
+ url: PROFILE_URL,
+ title: "Example Person",
+ markdown: "# Example Person",
+ source: {
+ provider: "acme",
+ recordId: "example-person",
+ recordType: "person",
+ },
+ ...(metadata === undefined ? {} : { metadata }),
+ },
+ };
+}
+
+const stubMeta = () =>
+ ({
+ id: "test",
+ url: PROFILE_URL,
+ options: { formats: [{ type: "markdown" }] },
+ internalOptions: { teamId: "team-1", orgId: "org-1" },
+ featureFlags: new Set(),
+ mock: null,
+ abort: { asSignal: () => undefined },
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ debug: vi.fn(),
+ error: vi.fn(),
+ child: vi.fn().mockReturnThis(),
+ },
+ }) as any;
+
+describe("Exchange provider metadata", () => {
+ const originalExchangeUrl = config.FIRE_EXCHANGE_URL;
+
+ beforeAll(() => {
+ config.FIRE_EXCHANGE_URL = "http://test-exchange";
+ });
+
+ afterAll(() => {
+ config.FIRE_EXCHANGE_URL = originalExchangeUrl;
+ });
+
+ beforeEach(() => {
+ mocks.robustFetch.mockReset();
+ });
+
+ describe("exchange engine", () => {
+ it("reports the matched waterfall step as the provider, with every step tried", async () => {
+ exchangeResponds(WATERFALL_RESPONSE);
+
+ const result = await scrapeURLWithExchange(stubMeta());
+
+ expect(result.exchange).toEqual({
+ accessEventId: "access-1",
+ provider: WATERFALL_PROVIDER,
+ });
+ });
+
+ it("reports a direct provider as its own single matched step", async () => {
+ exchangeResponds(directResponse({ cacheState: "miss" }));
+
+ const result = await scrapeURLWithExchange(stubMeta());
+
+ expect(result.exchange?.provider).toEqual({
+ id: "acme",
+ creditsCost: 12,
+ steps: [{ provider: "acme", status: "matched", creditsCost: 12 }],
+ });
+ });
+
+ it.each([
+ ["absent", undefined],
+ ["not an object", { enrichment: "matched" }],
+ [
+ "missing its matched source",
+ { enrichment: { status: "matched", steps: [] } },
+ ],
+ [
+ "an empty waterfall",
+ { enrichment: { source: { provider: "globex" }, steps: [] } },
+ ],
+ [
+ "a negative step cost",
+ {
+ enrichment: {
+ source: { provider: "globex" },
+ steps: [{ provider: "globex", status: "matched", creditsCost: -1 }],
+ },
+ },
+ ],
+ [
+ "missing a step status",
+ {
+ enrichment: {
+ source: { provider: "globex" },
+ steps: [{ provider: "globex", creditsCost: 12 }],
+ },
+ },
+ ],
+ ])(
+ "falls back to the direct-provider shape when enrichment metadata is %s",
+ async (_, metadata) => {
+ exchangeResponds(directResponse(metadata));
+
+ const result = await scrapeURLWithExchange(stubMeta());
+
+ expect(result.markdown).toBe("# Example Person");
+ expect(result.exchange?.provider).toEqual({
+ id: "acme",
+ creditsCost: 12,
+ steps: [{ provider: "acme", status: "matched", creditsCost: 12 }],
+ });
+ },
+ );
+ });
+
+ describe("scrape documents", () => {
+ const runScrape = (url: string, forceEngine: "exchange" | "fetch") =>
+ scrapeURL(
+ randomUUID(),
+ url,
+ scrapeOptions.parse({ formats: ["markdown"], maxAge: 0 }),
+ {
+ teamId: "exchange-provider-metadata-test",
+ orgId: null,
+ forceEngine,
+ },
+ new CostTracking(),
+ );
+
+ it(
+ "puts the provider on an Exchange-served document",
+ async () => {
+ exchangeResponds(WATERFALL_RESPONSE);
+
+ const result = await runScrape(PROFILE_URL, "exchange");
+
+ if (!result.success) throw result.error;
+ expect(result.document.metadata.provider).toEqual(WATERFALL_PROVIDER);
+ expect(result.exchange?.provider).toEqual(WATERFALL_PROVIDER);
+ },
+ scrapeTimeout,
+ );
+
+ itIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "leaves metadata.provider off documents the Exchange did not serve",
+ async () => {
+ const result = await runScrape(TEST_SUITE_WEBSITE, "fetch");
+
+ if (!result.success) throw result.error;
+ expect(result.exchange).toBeUndefined();
+ expect(result.document.metadata.provider).toBeUndefined();
+ },
+ scrapeTimeout,
+ );
+ });
+});
diff --git a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts
index f0712b0b9..f64f104c0 100644
--- a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts
+++ b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts
@@ -280,10 +280,8 @@ describe("Exchange routing", () => {
markdown: "# Example Person",
statusCode: 200,
exchange: {
- handled: true,
- creditsCost: 12,
accessEventId: "access-1",
- integrationId: "acme",
+ provider: { id: "acme", creditsCost: 12 },
},
});
});
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index bb7a3537e..727bc1f1c 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1436,6 +1436,14 @@ export const mapRequestSchema = strictWithMessage(
export type MapRequest = z.infer<typeof mapRequestSchema>;
export type MapRequestInput = z.input<typeof mapRequestSchema>;
+/** The third-party provider that served an Exchange scrape, what the access
+ * cost, and every provider tried for it in order. */
+export type DocumentProvider = {
+ id: string;
+ creditsCost: number;
+ steps: { provider: string; status: string; creditsCost?: number }[];
+};
+
export type Document = {
title?: string;
description?: string;
@@ -1552,6 +1560,7 @@ export type Document = {
indexId?: string; // ID used to store the document in the index (GCS)
concurrencyLimited?: boolean;
concurrencyQueueDurationMs?: number;
+ provider?: DocumentProvider;
// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;
};
serpResults?: {
diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.ts
index 92aee817f..1fb8d1ddc 100644
--- a/apps/api/src/lib/exchange.test.ts
+++ b/apps/api/src/lib/exchange.test.ts
@@ -557,21 +557,27 @@ describe("Exchange routing", () => {
expect(
getExchangeSuccessCredits({
- exchange: { handled: true, creditsCost: 12 },
+ exchange: {
+ provider: { id: "acme", creditsCost: 12, steps: [] },
+ },
statusCode: 200,
}),
).toBe(12);
expect(
getExchangeSuccessCredits({
- exchange: { handled: true, creditsCost: 0 },
+ exchange: {
+ provider: { id: "acme", creditsCost: 0, steps: [] },
+ },
statusCode: 304,
}),
).toBe(0);
expect(
getExchangeSuccessCredits({
- exchange: { handled: true, creditsCost: 12 },
+ exchange: {
+ provider: { id: "acme", creditsCost: 12, steps: [] },
+ },
statusCode: 404,
}),
).toBeNull();
diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.ts
index 7048bfdcf..9f7535b3c 100644
--- a/apps/api/src/lib/exchange.ts
+++ b/apps/api/src/lib/exchange.ts
@@ -2,7 +2,7 @@ import { fetch } from "undici";
import { z } from "zod";
import { config } from "../config";
-import type { FormatObject } from "../controllers/v2/types";
+import type { DocumentProvider, FormatObject } from "../controllers/v2/types";
import { hasLedgerAcceptance } from "../services/alexandria/terms";
import { type ErrorCodes, TransportableError } from "./error";
import { logger as rootLogger } from "./logger";
@@ -50,10 +50,8 @@ type RouteInput = {
};
export type ExchangeScrapeMetadata = {
- handled: true;
- creditsCost: number;
accessEventId?: string;
- integrationId?: string;
+ provider: DocumentProvider;
};
type ExchangeTerms = {
@@ -719,7 +717,7 @@ export function getExchangeSuccessCredits(input: {
exchange?: ExchangeScrapeMetadata;
statusCode?: number | null;
}): number | null {
- if (input.exchange?.handled !== true) {
+ if (input.exchange === undefined) {
return null;
}
@@ -732,7 +730,7 @@ export function getExchangeSuccessCredits(input: {
return null;
}
- return input.exchange.creditsCost;
+ return input.exchange.provider.creditsCost;
}
const EXCHANGE_BILLING_TIMEOUT_MS = 5_000;
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts
index b9c1e690d..a798a4a89 100644
--- a/apps/api/src/lib/scrape-billing.test.ts
+++ b/apps/api/src/lib/scrape-billing.test.ts
@@ -32,7 +32,7 @@ describe("calculateCreditsToBeBilled", () => {
{} as any,
undefined,
undefined,
- { handled: true, creditsCost: 12 },
+ { provider: { id: "acme", creditsCost: 12, steps: [] } },
);
expect(credits).toBe(12);
@@ -60,7 +60,7 @@ describe("calculateCreditsToBeBilled", () => {
{} as any,
undefined,
undefined,
- { handled: true, creditsCost: 12 },
+ { provider: { id: "acme", creditsCost: 12, steps: [] } },
);
expect(credits).toBe(16);
diff --git a/apps/api/src/scraper/scrapeURL/engines/exchange.ts b/apps/api/src/scraper/scrapeURL/engines/exchange.ts
index 1aa0f58c4..7d6638a16 100644
--- a/apps/api/src/scraper/scrapeURL/engines/exchange.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/exchange.ts
@@ -14,6 +14,21 @@ import { robustFetch } from "../lib/fetch";
import { safeMarkdownToHtml } from "./pdf/markdownToHtml";
import { EngineError, ExchangeRefusedError } from "../error";
+// The enrichment waterfall's record of which provider matched and every
+// provider it tried, in order.
+const exchangeEnrichmentSchema = z.object({
+ source: z.object({ provider: z.string() }),
+ steps: z
+ .array(
+ z.object({
+ provider: z.string(),
+ status: z.string(),
+ creditsCost: z.number().int().nonnegative().optional(),
+ }),
+ )
+ .min(1),
+});
+
const exchangeScrapeResponseSchema = z.union([
z
.object({
@@ -29,11 +44,17 @@ const exchangeScrapeResponseSchema = z.union([
description: z.string().optional(),
source: z
.object({
- provider: z.string().optional(),
+ provider: z.string(),
+ })
+ .passthrough(),
+ metadata: z
+ .object({
+ // Unreadable enrichment metadata reports the provider directly
+ // instead of failing a delivered scrape.
+ enrichment: exchangeEnrichmentSchema.optional().catch(undefined),
})
.passthrough()
.optional(),
- metadata: z.record(z.string(), z.unknown()).optional(),
markdown: z.string().optional(),
})
.passthrough(),
@@ -243,20 +264,24 @@ export async function scrapeURLWithExchange(
const responseLogContext = getExchangeResponseLogContext(
response.data.metadata,
);
+ const enrichment = response.data.metadata?.enrichment;
+ const providerId =
+ enrichment?.source.provider ?? response.data.source.provider;
logger.info("Exchange scrape completed", {
...requestLogContext,
...responseLogContext,
scrapeId: meta.id,
teamId: meta.internalOptions.teamId,
- integrationId: response.data.source?.provider,
+ integrationId: response.data.source.provider,
+ providerId,
accessEventId: response.accessEventId,
creditsCost: response.creditsCost,
durationMs: Date.now() - startTime,
});
setSpanAttributes(span, {
- "exchange.integration_id": response.data.source?.provider,
+ "exchange.integration_id": response.data.source.provider,
"exchange.credits_cost": response.creditsCost,
"exchange.cache_state": responseLogContext.cacheState,
"exchange.cache_age_ms": responseLogContext.cacheAgeMs,
@@ -276,14 +301,20 @@ export async function scrapeURLWithExchange(
contentType: "text/markdown",
proxyUsed: "basic",
exchange: {
- handled: true,
- creditsCost: response.creditsCost,
...(response.accessEventId === undefined
? {}
: { accessEventId: response.accessEventId }),
- ...(response.data.source?.provider === undefined
- ? {}
- : { integrationId: response.data.source.provider }),
+ provider: {
+ id: providerId,
+ creditsCost: response.creditsCost,
+ steps: enrichment?.steps ?? [
+ {
+ provider: providerId,
+ status: "matched",
+ creditsCost: response.creditsCost,
+ },
+ ],
+ },
},
};
} catch (error) {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index d7ca0cb96..b12b420a6 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1200,6 +1200,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
}
: {}),
postprocessorsUsed: engineResult.postprocessorsUsed,
+ provider: engineResult.exchange?.provider,
},
};