Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): attribute Alexandria provider calls to the discovery that surfaced them (#5312)

Nicolas253ee3f62495
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.test.ts b/apps/api/src/controllers/v2/scrape-alexandria.test.ts
index cbce0da53..d7eb788e1 100644
--- a/apps/api/src/controllers/v2/scrape-alexandria.test.ts
+++ b/apps/api/src/controllers/v2/scrape-alexandria.test.ts
@@ -8,6 +8,12 @@ const mocks = vi.hoisted(() => ({
retrieve: vi.fn(),
log: vi.fn(),
scrapeLog: vi.fn(),
+ discovery: vi.fn(),
+ recordDiscovered: vi.fn(),
+}));
+vi.mock("../../lib/alexandria-discovery", () => ({
+ discoveryForCalls: mocks.discovery,
+ recordDiscoveredTools: mocks.recordDiscovered,
}));
vi.mock("../../services/alexandria/retrieve", () => ({
REQUEST_ID_PATTERN: /^[A-Za-z0-9._:-]{1,128}$/,
@@ -68,6 +74,9 @@ beforeEach(() => {
flags = { exchangeRetrieve: true };
mocks.log.mockResolvedValue(undefined);
mocks.scrapeLog.mockResolvedValue(undefined);
+ mocks.discovery.mockImplementation(async (_teamId, calls: unknown[]) =>
+ calls.map(() => ({})),
+ );
mocks.retrieve.mockResolvedValue(
result([{ ...call, creditsCost: 0, data: {} }]),
);
@@ -360,3 +369,114 @@ it("does not ask for zero data retention by default", async () => {
),
);
});
+
+const findTools = {
+ provider: "firecrawl",
+ capability: "find-tools",
+ options: {
+ query: "podcast sponsors",
+ providers: ["particle"],
+ urls: ["https://a.example/", "https://b.example/"],
+ token: "secret-token",
+ },
+};
+
+it("logs find-tools options and result size, and remembers the tools it showed", async () => {
+ mocks.retrieve.mockResolvedValue(
+ result([
+ {
+ provider: "firecrawl",
+ capability: "find-tools",
+ creditsCost: 0,
+ data: {
+ level: "tools",
+ items: [
+ { provider: "particle", capability: "advertising/sponsors" },
+ { provider: "particle", capability: "podcasts/show" },
+ { provider: "particle", id: "particle" },
+ ],
+ },
+ },
+ ]),
+ );
+ const response = await request(app)
+ .post("/v2/scrape")
+ .send({ alexandria: findTools });
+ expect(response.status).toBe(200);
+ expect(mocks.recordDiscovered).toHaveBeenCalledWith(
+ "team",
+ "find_tools",
+ "scrape-1",
+ [
+ { provider: "particle", capability: "advertising/sponsors" },
+ { provider: "particle", capability: "podcasts/show" },
+ ],
+ );
+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));
+ expect(mocks.scrapeLog.mock.calls[0][0].options).toEqual({
+ alexandria: [
+ {
+ provider: "firecrawl",
+ capability: "find-tools",
+ options: {
+ query: "podcast sponsors",
+ providers: ["particle"],
+ urlCount: 2,
+ },
+ returned: 3,
+ },
+ ],
+ });
+});
+
+it("attributes each provider call to the discovery that showed it", async () => {
+ mocks.discovery.mockResolvedValue([
+ {
+ search: { id: "search-1", secondsBefore: 40 },
+ find_tools: { id: "scrape-0", secondsBefore: 12 },
+ },
+ ]);
+ await request(app).post("/v2/scrape").send({ alexandria: call });
+ expect(mocks.discovery).toHaveBeenCalledWith("team", [call]);
+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));
+ expect(mocks.scrapeLog.mock.calls[0][0].options).toEqual({
+ alexandria: [
+ {
+ provider: call.provider,
+ capability: call.capability,
+ discovery: {
+ search: { id: "search-1", secondsBefore: 40 },
+ find_tools: { id: "scrape-0", secondsBefore: 12 },
+ },
+ },
+ ],
+ });
+});
+
+it("still logs the call when the discovery lookup fails", async () => {
+ mocks.discovery.mockRejectedValue(new Error("Redis unavailable"));
+ await request(app).post("/v2/scrape").send({ alexandria: call });
+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));
+ expect(mocks.scrapeLog.mock.calls[0][0].options).toEqual({
+ alexandria: [{ provider: call.provider, capability: call.capability }],
+ });
+});
+
+it("neither reads nor records discovery under zero data retention", async () => {
+ mocks.retrieve.mockResolvedValue(
+ result([
+ {
+ provider: "firecrawl",
+ capability: "find-tools",
+ creditsCost: 0,
+ data: { items: [{ provider: "fred", capability: "series/search" }] },
+ },
+ ]),
+ );
+ await request(app)
+ .post("/v2/scrape")
+ .send({ alexandria: findTools, zeroDataRetention: true });
+ await vi.waitFor(() => expect(mocks.scrapeLog).toHaveBeenCalledTimes(1));
+ expect(mocks.discovery).not.toHaveBeenCalled();
+ expect(mocks.recordDiscovered).not.toHaveBeenCalled();
+});
diff --git a/apps/api/src/controllers/v2/scrape-alexandria.ts b/apps/api/src/controllers/v2/scrape-alexandria.ts
index 6299f5b63..e82fcbd5b 100644
--- a/apps/api/src/controllers/v2/scrape-alexandria.ts
+++ b/apps/api/src/controllers/v2/scrape-alexandria.ts
@@ -12,6 +12,10 @@ import { checkKeyFormatRestriction } from "../../lib/key-restriction";
import { orgIdFromAcuc } from "../../lib/team-org";
import { logProviderScrape, logRequest } from "../../services/logging/log_job";
import { markAlexandriaActivity } from "../../lib/alexandria-activity";
+import {
+ discoveryForCalls,
+ recordDiscoveredTools,
+} from "../../lib/alexandria-discovery";
import {
callsSchema,
callSchema,
@@ -23,6 +27,26 @@ import {
} from "../../services/alexandria/retrieve";
import type { RequestWithAuth } from "./types";
+const FIND_TOOLS_LOGGED_OPTIONS = [
+ "query",
+ "providers",
+ "categories",
+ "groups",
+ "capabilities",
+ "level",
+ "limit",
+ "offset",
+ "expand",
+];
+
+const isFindTools = (call: { provider?: string; capability?: string }) =>
+ call.provider === "firecrawl" && call.capability === "find-tools";
+
+const findToolsItems = (data: unknown): unknown[] => {
+ const items = (data as { items?: unknown } | null)?.items;
+ return Array.isArray(items) ? items : [];
+};
+
const providerScrapeSchema = z.strictObject({
alexandria: z.preprocess(
value => (Array.isArray(value) ? value : [value]),
@@ -126,6 +150,13 @@ export async function providerScrapeController(
.json({ success: false, error: "This endpoint is not available." });
const target = `alexandria:${body.alexandria.map(call => `${call.provider}/${call.capability}`).join(",")}`;
+ const discovery =
+ zeroDataRetention || body.__agentInterop
+ ? Promise.resolve(null)
+ : discoveryForCalls(req.auth.team_id, body.alexandria).catch(error => {
+ logger.warn("Alexandria discovery lookup failed", { error });
+ return null;
+ });
const startedAt = Date.now();
let result;
try {
@@ -176,8 +207,35 @@ export async function providerScrapeController(
const credits = served.success
? served.data.creditsCost
: failure?.creditsCost;
+ if (served.success && !zeroDataRetention)
+ recordDiscoveredTools(
+ req.auth.team_id,
+ "find_tools",
+ result.scrapeId,
+ served.data.results.flatMap(item =>
+ isFindTools(item)
+ ? findToolsItems(item.data).flatMap(tool => {
+ const { provider, capability } = (tool ?? {}) as {
+ provider?: unknown;
+ capability?: unknown;
+ };
+ return typeof provider === "string" &&
+ typeof capability === "string"
+ ? [{ provider, capability }]
+ : [];
+ })
+ : [],
+ ),
+ );
// Preserve parent/child ordering without delaying the provider response.
void (async () => {
+ // Bounded so a slow Redis never holds back the activity log.
+ const discovered = await Promise.race([
+ discovery,
+ new Promise<null>(resolve =>
+ setTimeout(() => resolve(null), 500).unref(),
+ ),
+ ]);
await logRequest({
id: result.scrapeId,
kind: "alexandria",
@@ -197,10 +255,35 @@ export async function providerScrapeController(
target,
team_id: req.auth.team_id,
options: {
- alexandria: body.alexandria.map(({ provider, capability }) => ({
- provider,
- capability,
- })),
+ alexandria: body.alexandria.map(
+ ({ provider, capability, options }, i) => ({
+ provider,
+ capability,
+ ...(isFindTools({ provider, capability })
+ ? {
+ options: {
+ ...Object.fromEntries(
+ Object.entries(options).filter(([key]) =>
+ FIND_TOOLS_LOGGED_OPTIONS.includes(key),
+ ),
+ ),
+ ...(Array.isArray(options.urls)
+ ? { urlCount: options.urls.length }
+ : {}),
+ },
+ ...(served.success
+ ? {
+ returned: findToolsItems(served.data.results[i]?.data)
+ .length,
+ }
+ : {}),
+ }
+ : {}),
+ ...(discovered && Object.keys(discovered[i]).length
+ ? { discovery: discovered[i] }
+ : {}),
+ }),
+ ),
},
time_taken: timeTaken,
credits_cost:
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 141bc9fb7..085fb0cba 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -60,6 +60,7 @@ import { requestOrigin } from "../../lib/request-origin";
import { isAgentInteropSecretValid } from "../../lib/agent-interop";
import { applyNotice, type Notice } from "../../lib/deprecations";
import { markAlexandriaActivity } from "../../lib/alexandria-activity";
+import { recordDiscoveredTools } from "../../lib/alexandria-discovery";
const RESEARCH_CATEGORY_NOTICE: Notice = {
message:
@@ -599,7 +600,15 @@ async function searchControllerInner(
scrapeful: result.shouldScrape,
});
- if (result.response.tools) markAlexandriaActivity(req.auth.team_id);
+ if (result.response.tools) {
+ markAlexandriaActivity(req.auth.team_id);
+ recordDiscoveredTools(
+ req.auth.team_id,
+ "search",
+ jobId,
+ result.response.tools,
+ );
+ }
return res.status(200).json({
success: true,
data: result.response,
diff --git a/apps/api/src/lib/__tests__/search-result-categories.test.ts b/apps/api/src/lib/__tests__/search-result-categories.test.ts
index 56ef94303..04bd98d45 100644
--- a/apps/api/src/lib/__tests__/search-result-categories.test.ts
+++ b/apps/api/src/lib/__tests__/search-result-categories.test.ts
@@ -2,6 +2,7 @@ import { describe, it, expect } from "vitest";
import {
collectSearchResultCategories,
countSearchResultsBySource,
+ type SearchResultType,
type SearchV2Response,
} from "../entities";
@@ -95,6 +96,23 @@ describe("collectSearchResultCategories", () => {
});
});
+ it("counts tools only when provider discovery ran", () => {
+ expect(
+ countSearchResultsBySource({ web: [web("https://a.example/")] }),
+ ).toEqual({
+ web: 1,
+ images: 0,
+ news: 0,
+ });
+ expect(
+ countSearchResultsBySource({
+ tools: [
+ { provider: "fred", capability: "series/search", description: "" },
+ ],
+ }),
+ ).toEqual({ web: 0, images: 0, news: 0, tools: 1 });
+ });
+
it("positions line up with the counts feedback bounds against", () => {
const response: SearchV2Response = {
web: [web("https://a.example/", "developer"), web("https://b.example/")],
@@ -107,7 +125,7 @@ describe("collectSearchResultCategories", () => {
for (const [source, byPosition] of Object.entries(categories)) {
for (const position of Object.keys(byPosition)) {
expect(Number(position)).toBeLessThanOrEqual(
- counts[source as keyof typeof counts],
+ counts[source as SearchResultType],
);
}
}
diff --git a/apps/api/src/lib/alexandria-discovery.test.ts b/apps/api/src/lib/alexandria-discovery.test.ts
new file mode 100644
index 000000000..7ec52fdb7
--- /dev/null
+++ b/apps/api/src/lib/alexandria-discovery.test.ts
@@ -0,0 +1,103 @@
+import { beforeEach, expect, it, vi } from "vitest";
+
+const store = vi.hoisted(() => new Map<string, string>());
+const expiries = vi.hoisted(() => new Map<string, unknown[]>());
+const failNextWrite = vi.hoisted(() => ({ value: false }));
+const warn = vi.hoisted(() => vi.fn());
+vi.mock("./logger", () => ({ logger: { warn } }));
+vi.mock("../services/rate-limiter", () => ({
+ redisRateLimitClient: {
+ pipeline: () => {
+ const writes: [string, string][] = [];
+ const pipeline = {
+ set: (key: string, value: string, ...expiry: unknown[]) => {
+ writes.push([key, value]);
+ expiries.set(key, expiry);
+ return pipeline;
+ },
+ exec: async () => {
+ if (failNextWrite.value) {
+ failNextWrite.value = false;
+ return writes.map(() => [new Error("OOM"), null]);
+ }
+ for (const [key, value] of writes) store.set(key, value);
+ return writes.map(() => [null, "OK"]);
+ },
+ };
+ return pipeline;
+ },
+ mget: async (keys: string[]) => keys.map(key => store.get(key) ?? null),
+ },
+}));
+import {
+ discoveryForCalls,
+ recordDiscoveredTools,
+} from "./alexandria-discovery";
+
+beforeEach(() => {
+ store.clear();
+ expiries.clear();
+ warn.mockClear();
+});
+
+it("attributes a call to the latest search and find-tools jobs that showed it", async () => {
+ const sponsors = { provider: "particle", capability: "advertising/sponsors" };
+ recordDiscoveredTools("team", "search", "search-1", [sponsors]);
+ recordDiscoveredTools("team", "find_tools", "scrape-1", [sponsors]);
+ recordDiscoveredTools("team", "find_tools", "scrape-2", [sponsors]);
+ await vi.waitFor(() => expect(store.size).toBe(2));
+ for (const key of store.keys()) {
+ expect(expiries.get(key)).toEqual(["EX", 3600]);
+ }
+
+ const [found, missing] = await discoveryForCalls("team", [
+ sponsors,
+ { provider: "fred", capability: "series/search" },
+ ]);
+ expect(found).toEqual({
+ search: { id: "search-1", secondsBefore: expect.any(Number) },
+ find_tools: { id: "scrape-2", secondsBefore: expect.any(Number) },
+ });
+ expect(missing).toEqual({});
+});
+
+it("keeps teams and provider/capability splits apart", async () => {
+ recordDiscoveredTools("other-team", "search", "search-1", [
+ { provider: "fred", capability: "series/search" },
+ ]);
+ recordDiscoveredTools("team", "search", "search-2", [
+ { provider: "a/b", capability: "c" },
+ ]);
+ await vi.waitFor(() => expect(store.size).toBe(2));
+ expect(
+ await discoveryForCalls("team", [
+ { provider: "fred", capability: "series/search" },
+ { provider: "a", capability: "b/c" },
+ ]),
+ ).toEqual([{}, {}]);
+});
+
+it("treats a malformed entry as a miss without dropping the others", async () => {
+ const tool = { provider: "fred", capability: "series/search" };
+ recordDiscoveredTools("team", "search", "search-1", [tool]);
+ await vi.waitFor(() => expect(store.size).toBe(1));
+ const [searchKey] = store.keys();
+ store.set(searchKey!.replace('"search"', '"find_tools"'), "not json");
+ expect(await discoveryForCalls("team", [tool])).toEqual([
+ { search: { id: "search-1", secondsBefore: expect.any(Number) } },
+ ]);
+});
+
+it("logs a write that Redis refused", async () => {
+ failNextWrite.value = true;
+ recordDiscoveredTools("team", "search", "search-1", [
+ { provider: "fred", capability: "series/search" },
+ ]);
+ await vi.waitFor(() =>
+ expect(warn).toHaveBeenCalledWith("Failed to record Alexandria discovery", {
+ error: expect.any(Error),
+ teamId: "team",
+ }),
+ );
+ expect(store.size).toBe(0);
+});
diff --git a/apps/api/src/lib/alexandria-discovery.ts b/apps/api/src/lib/alexandria-discovery.ts
new file mode 100644
index 000000000..b66690aaa
--- /dev/null
+++ b/apps/api/src/lib/alexandria-discovery.ts
@@ -0,0 +1,77 @@
+import { redisRateLimitClient } from "../services/rate-limiter";
+import { logger } from "./logger";
+
+const WINDOW_SECONDS = 3600;
+const SOURCES = ["search", "find_tools"] as const;
+type DiscoverySource = (typeof SOURCES)[number];
+type ToolRef = { provider: string; capability: string };
+export type CallDiscovery = Partial<
+ Record<DiscoverySource, { id: string; secondsBefore: number }>
+>;
+
+const discoveredKey = (
+ source: DiscoverySource,
+ teamId: string,
+ tool: ToolRef,
+) =>
+ `alexandria:discovered:${JSON.stringify([source, teamId, tool.provider, tool.capability])}`;
+
+/**
+ * Remembers for an hour that a search or find-tools job showed these tools to
+ * the team, so a later provider call can be attributed to it. Never rejects.
+ */
+export function recordDiscoveredTools(
+ teamId: string,
+ source: DiscoverySource,
+ jobId: string,
+ tools: ToolRef[],
+): void {
+ if (!tools.length) return;
+ const value = JSON.stringify({ id: jobId, at: Date.now() });
+ const pipeline = redisRateLimitClient.pipeline();
+ for (const tool of tools)
+ pipeline.set(
+ discoveredKey(source, teamId, tool),
+ value,
+ "EX",
+ WINDOW_SECONDS,
+ );
+ pipeline
+ .exec()
+ .then(results => {
+ const error = results?.find(([commandError]) => commandError)?.[0];
+ if (error) throw error;
+ })
+ .catch(error =>
+ logger.warn("Failed to record Alexandria discovery", { error, teamId }),
+ );
+}
+
+/** The latest search and find-tools jobs in the last hour that showed each call's tool to the team. */
+export async function discoveryForCalls(
+ teamId: string,
+ calls: ToolRef[],
+): Promise<CallDiscovery[]> {
+ const values = await redisRateLimitClient.mget(
+ calls.flatMap(call =>
+ SOURCES.map(source => discoveredKey(source, teamId, call)),
+ ),
+ );
+ const now = Date.now();
+ return calls.map((_, i) =>
+ Object.fromEntries(
+ SOURCES.flatMap((source, j) => {
+ const value = values[i * SOURCES.length + j];
+ if (!value) return [];
+ try {
+ const { id, at } = JSON.parse(value) as { id: string; at: number };
+ return [
+ [source, { id, secondsBefore: Math.round((now - at) / 1000) }],
+ ];
+ } catch {
+ return [];
+ }
+ }),
+ ),
+ );
+}
diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts
index 179f79f30..d0165a87f 100644
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@ -175,8 +175,11 @@ export interface SearchV2Response {
* group is present and non-negative — a source that was not requested, or that
* returned nothing, counts 0. Persisted per search so position-based feedback
* can be bounded exactly per source rather than against a combined total.
+ * `tools` is present only when provider discovery ran.
*/
-export type SearchResultCountsBySource = Record<SearchResultType, number>;
+export type SearchResultCountsBySource = Record<SearchResultType, number> & {
+ tools?: number;
+};
export function countSearchResultsBySource(
response: SearchV2Response,
@@ -185,6 +188,7 @@ export function countSearchResultsBySource(
web: response.web?.length ?? 0,
images: response.images?.length ?? 0,
news: response.news?.length ?? 0,
+ ...(response.tools ? { tools: response.tools.length } : {}),
};
}