Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): attribute Alexandria provider calls to the discovery that surfaced them (#5312)
Nicolas253ee3f62495
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.tsindex cbce0da53..d7eb788e1 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.test.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts@@ -8,6 +8,12 @@ const mocks = vi.hoisted(() => ({retrieve: vi.fn(),log: vi.fn(),scrapeLog: vi.fn(),+ discovery: vi.fn(),+ recordDiscovered: vi.fn(),+}));+vi.mock("../../lib/alexandria-discovery", () => ({+ discoveryForCalls: mocks.discovery,+ recordDiscoveredTools: mocks.recordDiscovered,}));vi.mock("../../services/alexandria/retrieve", () => ({REQUEST_ID_PATTERN: /^[A-Za-z0-9._:-]{1,128}$/,@@ -68,6 +74,9 @@ beforeEach(() => {flags = { exchangeRetrieve: true };mocks.log.mockResolvedValue(undefined);mocks.scrapeLog.mockResolvedValue(undefined);+ mocks.discovery.mockImplementation(async (_teamId, calls: unknown[]) =>+ calls.map(() => ({})),+ );mocks.retrieve.mockResolvedValue(result([{ ...call, creditsCost: 0, data: {} }]),);@@ -360,3 +369,114 @@ it("does not ask for zero data retention by default", async () => {),);});++const findTools = {+ provider: "firecrawl",+ capability: "find-tools",+ options: {+ query: "podcast sponsors",+ providers: ["particle"],+ urls: ["https://a.example/", "https://b.example/"],+ token: "secret-token",+ },+};++it("logs find-tools options and result size, and remembers the tools it showed", async () => {+ mocks.retrieve.mockResolvedValue(+ result([+ {+ provider: "firecrawl",+ capability: "find-tools",+ creditsCost: 0,+ data: {+ level: "tools",+ items: [+ { provider: "particle", capability: "advertising/sponsors" },+ { provider: "particle", capability: "podcasts/show" },+ { provider: "particle", id: "particle" },+ ],+ },+ },+ ]),+ );+ const response = await request(app)+ .post("/v2/scrape")+ .send({ alexandria: findTools });+ expect(response.status).toBe(200);+ expect(mocks.recordDiscovered).toHaveBeenCalledWith(+ "team",+ "find_tools",+ "scrape-1",+ [+ { provider: "particle", capability: "advertising/sponsors" },+ { provider: "particle", capability: "podcasts/show" },+ ],+ );+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));+ expect(mocks.scrapeLog.mock.calls[0][0].options).toEqual({+ alexandria: [+ {+ provider: "firecrawl",+ capability: "find-tools",+ options: {+ query: "podcast sponsors",+ providers: ["particle"],+ urlCount: 2,+ },+ returned: 3,+ },+ ],+ });+});++it("attributes each provider call to the discovery that showed it", async () => {+ mocks.discovery.mockResolvedValue([+ {+ search: { id: "search-1", secondsBefore: 40 },+ find_tools: { id: "scrape-0", secondsBefore: 12 },+ },+ ]);+ await request(app).post("/v2/scrape").send({ alexandria: call });+ expect(mocks.discovery).toHaveBeenCalledWith("team", [call]);+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));+ expect(mocks.scrapeLog.mock.calls[0][0].options).toEqual({+ alexandria: [+ {+ provider: call.provider,+ capability: call.capability,+ discovery: {+ search: { id: "search-1", secondsBefore: 40 },+ find_tools: { id: "scrape-0", secondsBefore: 12 },+ },+ },+ ],+ });+});++it("still logs the call when the discovery lookup fails", async () => {+ mocks.discovery.mockRejectedValue(new Error("Redis unavailable"));+ await request(app).post("/v2/scrape").send({ alexandria: call });+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));+ expect(mocks.scrapeLog.mock.calls[0][0].options).toEqual({+ alexandria: [{ provider: call.provider, capability: call.capability }],+ });+});++it("neither reads nor records discovery under zero data retention", async () => {+ mocks.retrieve.mockResolvedValue(+ result([+ {+ provider: "firecrawl",+ capability: "find-tools",+ creditsCost: 0,+ data: { items: [{ provider: "fred", capability: "series/search" }] },+ },+ ]),+ );+ await request(app)+ .post("/v2/scrape")+ .send({ alexandria: findTools, zeroDataRetention: true });+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));+ expect(mocks.discovery).not.toHaveBeenCalled();+ expect(mocks.recordDiscovered).not.toHaveBeenCalled();+});diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.tsindex 6299f5b63..e82fcbd5b 100644--- a/apps/api/src/controllers/v2/scrape-alexandria.ts+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts@@ -12,6 +12,10 @@ import { checkKeyFormatRestriction } from "../../lib/key-restriction";import { orgIdFromAcuc } from "../../lib/team-org";import { logProviderScrape, logRequest } from "../../services/logging/log_job";import { markAlexandriaActivity } from "../../lib/alexandria-activity";+import {+ discoveryForCalls,+ recordDiscoveredTools,+} from "../../lib/alexandria-discovery";import {callsSchema,callSchema,@@ -23,6 +27,26 @@ import {} from "../../services/alexandria/retrieve";import type { RequestWithAuth } from "./types";+const FIND_TOOLS_LOGGED_OPTIONS = [+ "query",+ "providers",+ "categories",+ "groups",+ "capabilities",+ "level",+ "limit",+ "offset",+ "expand",+];++const isFindTools = (call: { provider?: string; capability?: string }) =>+ call.provider === "firecrawl" && call.capability === "find-tools";++const findToolsItems = (data: unknown): unknown[] => {+ const items = (data as { items?: unknown } | null)?.items;+ return Array.isArray(items) ? items : [];+};+const providerScrapeSchema = z.strictObject({alexandria: z.preprocess(value => (Array.isArray(value) ? value : [value]),@@ -126,6 +150,13 @@ export async function providerScrapeController(.json({ success: false, error: "This endpoint is not available." });const target = `alexandria:${body.alexandria.map(call => `${call.provider}/${call.capability}`).join(",")}`;+ const discovery =+ zeroDataRetention || body.__agentInterop+ ? Promise.resolve(null)+ : discoveryForCalls(req.auth.team_id, body.alexandria).catch(error => {+ logger.warn("Alexandria discovery lookup failed", { error });+ return null;+ });const startedAt = Date.now();let result;try {@@ -176,8 +207,35 @@ export async function providerScrapeController(const credits = served.success? served.data.creditsCost: failure?.creditsCost;+ if (served.success && !zeroDataRetention)+ recordDiscoveredTools(+ req.auth.team_id,+ "find_tools",+ result.scrapeId,+ served.data.results.flatMap(item =>+ isFindTools(item)+ ? findToolsItems(item.data).flatMap(tool => {+ const { provider, capability } = (tool ?? {}) as {+ provider?: unknown;+ capability?: unknown;+ };+ return typeof provider === "string" &&+ typeof capability === "string"+ ? [{ provider, capability }]+ : [];+ })+ : [],+ ),+ );// Preserve parent/child ordering without delaying the provider response.void (async () => {+ // Bounded so a slow Redis never holds back the activity log.+ const discovered = await Promise.race([+ discovery,+ new Promise<null>(resolve =>+ setTimeout(() => resolve(null), 500).unref(),+ ),+ ]);await logRequest({id: result.scrapeId,kind: "alexandria",@@ -197,10 +255,35 @@ export async function providerScrapeController(target,team_id: req.auth.team_id,options: {- alexandria: body.alexandria.map(({ provider, capability }) => ({- provider,- capability,- })),+ alexandria: body.alexandria.map(+ ({ provider, capability, options }, i) => ({+ provider,+ capability,+ ...(isFindTools({ provider, capability })+ ? {+ options: {+ ...Object.fromEntries(+ Object.entries(options).filter(([key]) =>+ FIND_TOOLS_LOGGED_OPTIONS.includes(key),+ ),+ ),+ ...(Array.isArray(options.urls)+ ? { urlCount: options.urls.length }+ : {}),+ },+ ...(served.success+ ? {+ returned: findToolsItems(served.data.results[i]?.data)+ .length,+ }+ : {}),+ }+ : {}),+ ...(discovered && Object.keys(discovered[i]).length+ ? { discovery: discovered[i] }+ : {}),+ }),+ ),},time_taken: timeTaken,credits_cost:diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 141bc9fb7..085fb0cba 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -60,6 +60,7 @@ import { requestOrigin } from "../../lib/request-origin";import { isAgentInteropSecretValid } from "../../lib/agent-interop";import { applyNotice, type Notice } from "../../lib/deprecations";import { markAlexandriaActivity } from "../../lib/alexandria-activity";+import { recordDiscoveredTools } from "../../lib/alexandria-discovery";const RESEARCH_CATEGORY_NOTICE: Notice = {message:@@ -599,7 +600,15 @@ async function searchControllerInner(scrapeful: result.shouldScrape,});- if (result.response.tools) markAlexandriaActivity(req.auth.team_id);+ if (result.response.tools) {+ markAlexandriaActivity(req.auth.team_id);+ recordDiscoveredTools(+ req.auth.team_id,+ "search",+ jobId,+ result.response.tools,+ );+ }return res.status(200).json({success: true,data: result.response,diff --git a/apps/api/src/lib/__tests__/search-result-categories.test.ts b/apps/api/src/lib/__tests__/search-result-categories.test.tsindex 56ef94303..04bd98d45 100644--- a/apps/api/src/lib/__tests__/search-result-categories.test.ts+++ b/apps/api/src/lib/__tests__/search-result-categories.test.ts@@ -2,6 +2,7 @@ import { describe, it, expect } from "vitest";import {collectSearchResultCategories,countSearchResultsBySource,+ type SearchResultType,type SearchV2Response,} from "../entities";@@ -95,6 +96,23 @@ describe("collectSearchResultCategories", () => {});});+ it("counts tools only when provider discovery ran", () => {+ expect(+ countSearchResultsBySource({ web: [web("https://a.example/")] }),+ ).toEqual({+ web: 1,+ images: 0,+ news: 0,+ });+ expect(+ countSearchResultsBySource({+ tools: [+ { provider: "fred", capability: "series/search", description: "" },+ ],+ }),+ ).toEqual({ web: 0, images: 0, news: 0, tools: 1 });+ });+it("positions line up with the counts feedback bounds against", () => {const response: SearchV2Response = {web: [web("https://a.example/", "developer"), web("https://b.example/")],@@ -107,7 +125,7 @@ describe("collectSearchResultCategories", () => {for (const [source, byPosition] of Object.entries(categories)) {for (const position of Object.keys(byPosition)) {expect(Number(position)).toBeLessThanOrEqual(- counts[source as keyof typeof counts],+ counts[source as SearchResultType],);}}diff --git a/apps/api/src/lib/alexandria-discovery.test.ts b/apps/api/src/lib/alexandria-discovery.test.tsnew file mode 100644index 000000000..7ec52fdb7--- /dev/null+++ b/apps/api/src/lib/alexandria-discovery.test.ts@@ -0,0 +1,103 @@+import { beforeEach, expect, it, vi } from "vitest";++const store = vi.hoisted(() => new Map<string, string>());+const expiries = vi.hoisted(() => new Map<string, unknown[]>());+const failNextWrite = vi.hoisted(() => ({ value: false }));+const warn = vi.hoisted(() => vi.fn());+vi.mock("./logger", () => ({ logger: { warn } }));+vi.mock("../services/rate-limiter", () => ({+ redisRateLimitClient: {+ pipeline: () => {+ const writes: [string, string][] = [];+ const pipeline = {+ set: (key: string, value: string, ...expiry: unknown[]) => {+ writes.push([key, value]);+ expiries.set(key, expiry);+ return pipeline;+ },+ exec: async () => {+ if (failNextWrite.value) {+ failNextWrite.value = false;+ return writes.map(() => [new Error("OOM"), null]);+ }+ for (const [key, value] of writes) store.set(key, value);+ return writes.map(() => [null, "OK"]);+ },+ };+ return pipeline;+ },+ mget: async (keys: string[]) => keys.map(key => store.get(key) ?? null),+ },+}));+import {+ discoveryForCalls,+ recordDiscoveredTools,+} from "./alexandria-discovery";++beforeEach(() => {+ store.clear();+ expiries.clear();+ warn.mockClear();+});++it("attributes a call to the latest search and find-tools jobs that showed it", async () => {+ const sponsors = { provider: "particle", capability: "advertising/sponsors" };+ recordDiscoveredTools("team", "search", "search-1", [sponsors]);+ recordDiscoveredTools("team", "find_tools", "scrape-1", [sponsors]);+ recordDiscoveredTools("team", "find_tools", "scrape-2", [sponsors]);+ await vi.waitFor(() => expect(store.size).toBe(2));+ for (const key of store.keys()) {+ expect(expiries.get(key)).toEqual(["EX", 3600]);+ }++ const [found, missing] = await discoveryForCalls("team", [+ sponsors,+ { provider: "fred", capability: "series/search" },+ ]);+ expect(found).toEqual({+ search: { id: "search-1", secondsBefore: expect.any(Number) },+ find_tools: { id: "scrape-2", secondsBefore: expect.any(Number) },+ });+ expect(missing).toEqual({});+});++it("keeps teams and provider/capability splits apart", async () => {+ recordDiscoveredTools("other-team", "search", "search-1", [+ { provider: "fred", capability: "series/search" },+ ]);+ recordDiscoveredTools("team", "search", "search-2", [+ { provider: "a/b", capability: "c" },+ ]);+ await vi.waitFor(() => expect(store.size).toBe(2));+ expect(+ await discoveryForCalls("team", [+ { provider: "fred", capability: "series/search" },+ { provider: "a", capability: "b/c" },+ ]),+ ).toEqual([{}, {}]);+});++it("treats a malformed entry as a miss without dropping the others", async () => {+ const tool = { provider: "fred", capability: "series/search" };+ recordDiscoveredTools("team", "search", "search-1", [tool]);+ await vi.waitFor(() => expect(store.size).toBe(1));+ const [searchKey] = store.keys();+ store.set(searchKey!.replace('"search"', '"find_tools"'), "not json");+ expect(await discoveryForCalls("team", [tool])).toEqual([+ { search: { id: "search-1", secondsBefore: expect.any(Number) } },+ ]);+});++it("logs a write that Redis refused", async () => {+ failNextWrite.value = true;+ recordDiscoveredTools("team", "search", "search-1", [+ { provider: "fred", capability: "series/search" },+ ]);+ await vi.waitFor(() =>+ expect(warn).toHaveBeenCalledWith("Failed to record Alexandria discovery", {+ error: expect.any(Error),+ teamId: "team",+ }),+ );+ expect(store.size).toBe(0);+});diff --git a/apps/api/src/lib/alexandria-discovery.ts b/apps/api/src/lib/alexandria-discovery.tsnew file mode 100644index 000000000..b66690aaa--- /dev/null+++ b/apps/api/src/lib/alexandria-discovery.ts@@ -0,0 +1,77 @@+import { redisRateLimitClient } from "../services/rate-limiter";+import { logger } from "./logger";++const WINDOW_SECONDS = 3600;+const SOURCES = ["search", "find_tools"] as const;+type DiscoverySource = (typeof SOURCES)[number];+type ToolRef = { provider: string; capability: string };+export type CallDiscovery = Partial<+ Record<DiscoverySource, { id: string; secondsBefore: number }>+>;++const discoveredKey = (+ source: DiscoverySource,+ teamId: string,+ tool: ToolRef,+) =>+ `alexandria:discovered:${JSON.stringify([source, teamId, tool.provider, tool.capability])}`;++/**+ * Remembers for an hour that a search or find-tools job showed these tools to+ * the team, so a later provider call can be attributed to it. Never rejects.+ */+export function recordDiscoveredTools(+ teamId: string,+ source: DiscoverySource,+ jobId: string,+ tools: ToolRef[],+): void {+ if (!tools.length) return;+ const value = JSON.stringify({ id: jobId, at: Date.now() });+ const pipeline = redisRateLimitClient.pipeline();+ for (const tool of tools)+ pipeline.set(+ discoveredKey(source, teamId, tool),+ value,+ "EX",+ WINDOW_SECONDS,+ );+ pipeline+ .exec()+ .then(results => {+ const error = results?.find(([commandError]) => commandError)?.[0];+ if (error) throw error;+ })+ .catch(error =>+ logger.warn("Failed to record Alexandria discovery", { error, teamId }),+ );+}++/** The latest search and find-tools jobs in the last hour that showed each call's tool to the team. */+export async function discoveryForCalls(+ teamId: string,+ calls: ToolRef[],+): Promise<CallDiscovery[]> {+ const values = await redisRateLimitClient.mget(+ calls.flatMap(call =>+ SOURCES.map(source => discoveredKey(source, teamId, call)),+ ),+ );+ const now = Date.now();+ return calls.map((_, i) =>+ Object.fromEntries(+ SOURCES.flatMap((source, j) => {+ const value = values[i * SOURCES.length + j];+ if (!value) return [];+ try {+ const { id, at } = JSON.parse(value) as { id: string; at: number };+ return [+ [source, { id, secondsBefore: Math.round((now - at) / 1000) }],+ ];+ } catch {+ return [];+ }+ }),+ ),+ );+}diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.tsindex 179f79f30..d0165a87f 100644--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@ -175,8 +175,11 @@ export interface SearchV2Response {* group is present and non-negative — a source that was not requested, or that* returned nothing, counts 0. Persisted per search so position-based feedback* can be bounded exactly per source rather than against a combined total.+ * `tools` is present only when provider discovery ran.*/-export type SearchResultCountsBySource = Record<SearchResultType, number>;+export type SearchResultCountsBySource = Record<SearchResultType, number> & {+ tools?: number;+};export function countSearchResultsBySource(response: SearchV2Response,@@ -185,6 +188,7 @@ export function countSearchResultsBySource(web: response.web?.length ?? 0,images: response.images?.length ?? 0,news: response.news?.length ?? 0,+ ...(response.tools ? { tools: response.tools.length } : {}),};}