Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): report the serving provider as metadata.provider on Exchange scrapes (#5000)
Gergő Móricza2833245bee7
diff --git a/apps/api/src/__tests__/snips/v2/exchange-provider-metadata.test.ts b/apps/api/src/__tests__/snips/v2/exchange-provider-metadata.test.tsnew file mode 100644index 000000000..e0b09f1c8--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/exchange-provider-metadata.test.ts@@ -0,0 +1,238 @@+import { randomUUID } from "node:crypto";+import { config } from "../../../config";+import { scrapeOptions } from "../../../controllers/v2/types";+import { CostTracking } from "../../../lib/cost-tracking";+import { scrapeURL } from "../../../scraper/scrapeURL";+import { scrapeURLWithExchange } from "../../../scraper/scrapeURL/engines/exchange";+import { ALLOW_TEST_SUITE_WEBSITE, itIf, TEST_SUITE_WEBSITE } from "../lib";+import { scrapeTimeout } from "./lib";++const mocks = vi.hoisted(() => ({ robustFetch: vi.fn() }));++vi.mock("../../../scraper/scrapeURL/lib/fetch", () => ({+ robustFetch: mocks.robustFetch,+}));++const PROFILE_URL = "https://profiles.example/in/example-person";++const WATERFALL_RESPONSE = {+ success: true,+ accessEventId: "access-1",+ creditsCost: 30,+ data: {+ url: PROFILE_URL,+ title: "Example Person",+ markdown: "# Example Person",+ source: {+ provider: "firecrawl-enrich",+ recordId: PROFILE_URL,+ recordType: "professional_profile",+ },+ metadata: {+ canonicalUrl: PROFILE_URL,+ targetKind: "person",+ enrichment: {+ status: "matched",+ preferenceVersion: 1,+ source: { provider: "globex", capability: "person-profile" },+ steps: [+ {+ provider: "acme",+ capability: "person-profile",+ requestId: "step-1",+ status: "not_found",+ creditsCost: 10,+ },+ {+ provider: "globex",+ capability: "person-profile",+ requestId: "step-2",+ status: "matched",+ creditsCost: 20,+ },+ ],+ providerCredits: 30,+ },+ },+ },+};++const WATERFALL_PROVIDER = {+ id: "globex",+ creditsCost: 30,+ steps: [+ { provider: "acme", status: "not_found", creditsCost: 10 },+ { provider: "globex", status: "matched", creditsCost: 20 },+ ],+};++// Parses with the engine's own schema, as the real robustFetch does.+function exchangeResponds(response: unknown) {+ mocks.robustFetch.mockImplementation(async ({ schema }) =>+ schema.parse(response),+ );+}++function directResponse(metadata?: unknown) {+ return {+ success: true,+ accessEventId: "access-2",+ creditsCost: 12,+ data: {+ url: PROFILE_URL,+ title: "Example Person",+ markdown: "# Example Person",+ source: {+ provider: "acme",+ recordId: "example-person",+ recordType: "person",+ },+ ...(metadata === undefined ? {} : { metadata }),+ },+ };+}++const stubMeta = () =>+ ({+ id: "test",+ url: PROFILE_URL,+ options: { formats: [{ type: "markdown" }] },+ internalOptions: { teamId: "team-1", orgId: "org-1" },+ featureFlags: new Set(),+ mock: null,+ abort: { asSignal: () => undefined },+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ debug: vi.fn(),+ error: vi.fn(),+ child: vi.fn().mockReturnThis(),+ },+ }) as any;++describe("Exchange provider metadata", () => {+ const originalExchangeUrl = config.FIRE_EXCHANGE_URL;++ beforeAll(() => {+ config.FIRE_EXCHANGE_URL = "http://test-exchange";+ });++ afterAll(() => {+ config.FIRE_EXCHANGE_URL = originalExchangeUrl;+ });++ beforeEach(() => {+ mocks.robustFetch.mockReset();+ });++ describe("exchange engine", () => {+ it("reports the matched waterfall step as the provider, with every step tried", async () => {+ exchangeResponds(WATERFALL_RESPONSE);++ const result = await scrapeURLWithExchange(stubMeta());++ expect(result.exchange).toEqual({+ accessEventId: "access-1",+ provider: WATERFALL_PROVIDER,+ });+ });++ it("reports a direct provider as its own single matched step", async () => {+ exchangeResponds(directResponse({ cacheState: "miss" }));++ const result = await scrapeURLWithExchange(stubMeta());++ expect(result.exchange?.provider).toEqual({+ id: "acme",+ creditsCost: 12,+ steps: [{ provider: "acme", status: "matched", creditsCost: 12 }],+ });+ });++ it.each([+ ["absent", undefined],+ ["not an object", { enrichment: "matched" }],+ [+ "missing its matched source",+ { enrichment: { status: "matched", steps: [] } },+ ],+ [+ "an empty waterfall",+ { enrichment: { source: { provider: "globex" }, steps: [] } },+ ],+ [+ "a negative step cost",+ {+ enrichment: {+ source: { provider: "globex" },+ steps: [{ provider: "globex", status: "matched", creditsCost: -1 }],+ },+ },+ ],+ [+ "missing a step status",+ {+ enrichment: {+ source: { provider: "globex" },+ steps: [{ provider: "globex", creditsCost: 12 }],+ },+ },+ ],+ ])(+ "falls back to the direct-provider shape when enrichment metadata is %s",+ async (_, metadata) => {+ exchangeResponds(directResponse(metadata));++ const result = await scrapeURLWithExchange(stubMeta());++ expect(result.markdown).toBe("# Example Person");+ expect(result.exchange?.provider).toEqual({+ id: "acme",+ creditsCost: 12,+ steps: [{ provider: "acme", status: "matched", creditsCost: 12 }],+ });+ },+ );+ });++ describe("scrape documents", () => {+ const runScrape = (url: string, forceEngine: "exchange" | "fetch") =>+ scrapeURL(+ randomUUID(),+ url,+ scrapeOptions.parse({ formats: ["markdown"], maxAge: 0 }),+ {+ teamId: "exchange-provider-metadata-test",+ orgId: null,+ forceEngine,+ },+ new CostTracking(),+ );++ it(+ "puts the provider on an Exchange-served document",+ async () => {+ exchangeResponds(WATERFALL_RESPONSE);++ const result = await runScrape(PROFILE_URL, "exchange");++ if (!result.success) throw result.error;+ expect(result.document.metadata.provider).toEqual(WATERFALL_PROVIDER);+ expect(result.exchange?.provider).toEqual(WATERFALL_PROVIDER);+ },+ scrapeTimeout,+ );++ itIf(ALLOW_TEST_SUITE_WEBSITE)(+ "leaves metadata.provider off documents the Exchange did not serve",+ async () => {+ const result = await runScrape(TEST_SUITE_WEBSITE, "fetch");++ if (!result.success) throw result.error;+ expect(result.exchange).toBeUndefined();+ expect(result.document.metadata.provider).toBeUndefined();+ },+ scrapeTimeout,+ );+ });+});diff --git a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts b/apps/api/src/__tests__/snips/v2/exchange-routing.test.tsindex f0712b0b9..f64f104c0 100644--- a/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts+++ b/apps/api/src/__tests__/snips/v2/exchange-routing.test.ts@@ -280,10 +280,8 @@ describe("Exchange routing", () => {markdown: "# Example Person",statusCode: 200,exchange: {- handled: true,- creditsCost: 12,accessEventId: "access-1",- integrationId: "acme",+ provider: { id: "acme", creditsCost: 12 },},});});diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex bb7a3537e..727bc1f1c 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1436,6 +1436,14 @@ export const mapRequestSchema = strictWithMessage(export type MapRequest = z.infer<typeof mapRequestSchema>;export type MapRequestInput = z.input<typeof mapRequestSchema>;+/** The third-party provider that served an Exchange scrape, what the access+ * cost, and every provider tried for it in order. */+export type DocumentProvider = {+ id: string;+ creditsCost: number;+ steps: { provider: string; status: string; creditsCost?: number }[];+};+export type Document = {title?: string;description?: string;@@ -1552,6 +1560,7 @@ export type Document = {indexId?: string; // ID used to store the document in the index (GCS)concurrencyLimited?: boolean;concurrencyQueueDurationMs?: number;+ provider?: DocumentProvider;// [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;};serpResults?: {diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.tsindex 92aee817f..1fb8d1ddc 100644--- a/apps/api/src/lib/exchange.test.ts+++ b/apps/api/src/lib/exchange.test.ts@@ -557,21 +557,27 @@ describe("Exchange routing", () => {expect(getExchangeSuccessCredits({- exchange: { handled: true, creditsCost: 12 },+ exchange: {+ provider: { id: "acme", creditsCost: 12, steps: [] },+ },statusCode: 200,}),).toBe(12);expect(getExchangeSuccessCredits({- exchange: { handled: true, creditsCost: 0 },+ exchange: {+ provider: { id: "acme", creditsCost: 0, steps: [] },+ },statusCode: 304,}),).toBe(0);expect(getExchangeSuccessCredits({- exchange: { handled: true, creditsCost: 12 },+ exchange: {+ provider: { id: "acme", creditsCost: 12, steps: [] },+ },statusCode: 404,}),).toBeNull();diff --git a/apps/api/src/lib/exchange.ts b/apps/api/src/lib/exchange.tsindex 7048bfdcf..9f7535b3c 100644--- a/apps/api/src/lib/exchange.ts+++ b/apps/api/src/lib/exchange.ts@@ -2,7 +2,7 @@ import { fetch } from "undici";import { z } from "zod";import { config } from "../config";-import type { FormatObject } from "../controllers/v2/types";+import type { DocumentProvider, FormatObject } from "../controllers/v2/types";import { hasLedgerAcceptance } from "../services/alexandria/terms";import { type ErrorCodes, TransportableError } from "./error";import { logger as rootLogger } from "./logger";@@ -50,10 +50,8 @@ type RouteInput = {};export type ExchangeScrapeMetadata = {- handled: true;- creditsCost: number;accessEventId?: string;- integrationId?: string;+ provider: DocumentProvider;};type ExchangeTerms = {@@ -719,7 +717,7 @@ export function getExchangeSuccessCredits(input: {exchange?: ExchangeScrapeMetadata;statusCode?: number | null;}): number | null {- if (input.exchange?.handled !== true) {+ if (input.exchange === undefined) {return null;}@@ -732,7 +730,7 @@ export function getExchangeSuccessCredits(input: {return null;}- return input.exchange.creditsCost;+ return input.exchange.provider.creditsCost;}const EXCHANGE_BILLING_TIMEOUT_MS = 5_000;diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex b9c1e690d..a798a4a89 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -32,7 +32,7 @@ describe("calculateCreditsToBeBilled", () => {{} as any,undefined,undefined,- { handled: true, creditsCost: 12 },+ { provider: { id: "acme", creditsCost: 12, steps: [] } },);expect(credits).toBe(12);@@ -60,7 +60,7 @@ describe("calculateCreditsToBeBilled", () => {{} as any,undefined,undefined,- { handled: true, creditsCost: 12 },+ { provider: { id: "acme", creditsCost: 12, steps: [] } },);expect(credits).toBe(16);diff --git a/apps/api/src/scraper/scrapeURL/engines/exchange.ts b/apps/api/src/scraper/scrapeURL/engines/exchange.tsindex 1aa0f58c4..7d6638a16 100644--- a/apps/api/src/scraper/scrapeURL/engines/exchange.ts+++ b/apps/api/src/scraper/scrapeURL/engines/exchange.ts@@ -14,6 +14,21 @@ import { robustFetch } from "../lib/fetch";import { safeMarkdownToHtml } from "./pdf/markdownToHtml";import { EngineError, ExchangeRefusedError } from "../error";+// The enrichment waterfall's record of which provider matched and every+// provider it tried, in order.+const exchangeEnrichmentSchema = z.object({+ source: z.object({ provider: z.string() }),+ steps: z+ .array(+ z.object({+ provider: z.string(),+ status: z.string(),+ creditsCost: z.number().int().nonnegative().optional(),+ }),+ )+ .min(1),+});+const exchangeScrapeResponseSchema = z.union([z.object({@@ -29,11 +44,17 @@ const exchangeScrapeResponseSchema = z.union([description: z.string().optional(),source: z.object({- provider: z.string().optional(),+ provider: z.string(),+ })+ .passthrough(),+ metadata: z+ .object({+ // Unreadable enrichment metadata reports the provider directly+ // instead of failing a delivered scrape.+ enrichment: exchangeEnrichmentSchema.optional().catch(undefined),}).passthrough().optional(),- metadata: z.record(z.string(), z.unknown()).optional(),markdown: z.string().optional(),}).passthrough(),@@ -243,20 +264,24 @@ export async function scrapeURLWithExchange(const responseLogContext = getExchangeResponseLogContext(response.data.metadata,);+ const enrichment = response.data.metadata?.enrichment;+ const providerId =+ enrichment?.source.provider ?? response.data.source.provider;logger.info("Exchange scrape completed", {...requestLogContext,...responseLogContext,scrapeId: meta.id,teamId: meta.internalOptions.teamId,- integrationId: response.data.source?.provider,+ integrationId: response.data.source.provider,+ providerId,accessEventId: response.accessEventId,creditsCost: response.creditsCost,durationMs: Date.now() - startTime,});setSpanAttributes(span, {- "exchange.integration_id": response.data.source?.provider,+ "exchange.integration_id": response.data.source.provider,"exchange.credits_cost": response.creditsCost,"exchange.cache_state": responseLogContext.cacheState,"exchange.cache_age_ms": responseLogContext.cacheAgeMs,@@ -276,14 +301,20 @@ export async function scrapeURLWithExchange(contentType: "text/markdown",proxyUsed: "basic",exchange: {- handled: true,- creditsCost: response.creditsCost,...(response.accessEventId === undefined? {}: { accessEventId: response.accessEventId }),- ...(response.data.source?.provider === undefined- ? {}- : { integrationId: response.data.source.provider }),+ provider: {+ id: providerId,+ creditsCost: response.creditsCost,+ steps: enrichment?.steps ?? [+ {+ provider: providerId,+ status: "matched",+ creditsCost: response.creditsCost,+ },+ ],+ },},};} catch (error) {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex d7ca0cb96..b12b420a6 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -1200,6 +1200,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {}: {}),postprocessorsUsed: engineResult.postprocessorsUsed,+ provider: engineResult.exchange?.provider,},};