Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(search): log the served tool list and fusion outcome per /v2/search (#5641)

Rakshith Ramprakashc9697e6b426b
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 0327c00fb..5e898a606 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -716,6 +716,7 @@ async function searchControllerInner(
markAlexandriaActivity(req.auth.team_id);
}
if (result.response.tools) {
+ setSpanAttributes(span, { "search.tool_fusion": result.toolFusion });
recordDiscoveredTools(
req.auth.team_id,
result.toolsAuto ? "auto_search" : "search",
diff --git a/apps/api/src/search/alexandria.test.ts b/apps/api/src/search/alexandria.test.ts
index ababcfe90..a67aa5509 100644
--- a/apps/api/src/search/alexandria.test.ts
+++ b/apps/api/src/search/alexandria.test.ts
@@ -283,3 +283,35 @@ it("reports no hidden tools when the Exchange hid nothing", async () => {
expect(result.items).toHaveLength(1);
expect(result).not.toHaveProperty("usOnlyExcluded");
});
+
+it("numbers each lookup's Exchange request id from the caller's prefix", async () => {
+ request.mockResolvedValue({
+ status: 200,
+ body: { success: true, creditsCost: 0, data: { items: [tool] } },
+ });
+ const exchangeRequestIds = {
+ prefix: "search-1:tools:raw",
+ sent: [] as string[],
+ };
+ await discoverTools(
+ { ...input, queries: ["first", "second"], exchangeRequestIds },
+ logger,
+ );
+ expect(request.mock.calls.map(([args]) => args.requestId).sort()).toEqual([
+ "search-1:tools:raw:1",
+ "search-1:tools:raw:2",
+ ]);
+ expect(exchangeRequestIds.sent.sort()).toEqual([
+ "search-1:tools:raw:1",
+ "search-1:tools:raw:2",
+ ]);
+});
+
+it("sends no request id when the caller passes none", async () => {
+ request.mockResolvedValue({
+ status: 200,
+ body: { success: true, creditsCost: 0, data: { items: [tool] } },
+ });
+ await discoverTools(input, logger);
+ expect(request.mock.calls[0][0].requestId).toBeUndefined();
+});
diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.ts
index 09babbdba..128a1b400 100644
--- a/apps/api/src/search/alexandria.ts
+++ b/apps/api/src/search/alexandria.ts
@@ -87,6 +87,8 @@ export async function discoverTools(
connectors?: boolean;
/** The team may only be served by US-based providers. */
usOnly?: boolean;
+ /** Names each find-tools lookup `<prefix>:<n>` and appends the id to `sent` as it goes out. */
+ exchangeRequestIds?: { prefix: string; sent: string[] };
},
logger: Logger,
): Promise<ToolDiscovery> {
@@ -102,10 +104,15 @@ export async function discoverTools(
return deadline - Date.now();
};
const lookup = async (options: Record<string, unknown>, deadline: number) => {
+ const timeoutMs = remaining(deadline);
+ const ids = input.exchangeRequestIds;
+ const requestId = ids ? `${ids.prefix}:${ids.sent.length + 1}` : undefined;
+ if (requestId) ids?.sent.push(requestId);
const result = await exchangeRequest({
teamId: input.teamId,
path: "/v1/retrieve",
- timeoutMs: remaining(deadline),
+ requestId,
+ timeoutMs,
maximumCredits: 0,
connectors: input.connectors === true,
usOnly: input.usOnly === true,
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.ts
index af26e700e..c66feea59 100644
--- a/apps/api/src/search/execute.test.ts
+++ b/apps/api/src/search/execute.test.ts
@@ -76,6 +76,11 @@ vi.mock("./safe-search", () => ({
import { executeSearch } from "./execute";
import { searchRequestSchema } from "../controllers/v2/types";
+import { logger as rootLogger } from "../lib/logger";
+
+const impressions = vi
+ .spyOn(rootLogger, "info")
+ .mockImplementation(() => rootLogger);
const logger = {
info: vi.fn(),
@@ -413,6 +418,100 @@ describe("executeSearch tool intent", () => {
);
});
+ it("logs the served tools with their rank, source and fit rejections", async () => {
+ mocks.inferToolIntent.mockResolvedValue(intent);
+ mocks.discoverTools.mockImplementation(
+ async (input: {
+ queries?: string[];
+ exchangeRequestIds?: { prefix: string; sent: string[] };
+ }) => {
+ input.exchangeRequestIds?.sent.push(
+ `${input.exchangeRequestIds.prefix}:1`,
+ );
+ return input.queries?.[0] === "who owns apple stock"
+ ? {
+ items: [tool("a"), tool("b"), tool("c"), tool("domain")],
+ semanticMatches: 3,
+ }
+ : {
+ items: [tool("x"), tool("c"), tool("y"), tool("site")],
+ semanticMatches: 3,
+ };
+ },
+ );
+ mocks.verifyTools.mockImplementation(async (_query, items) =>
+ items.filter(
+ (item: { provider: string }) =>
+ item.provider !== "y" && item.provider !== "site",
+ ),
+ );
+ const result = await fused("who owns apple stock");
+ expect(result.toolFusion).toBe(true);
+ expect(mocks.discoverTools).toHaveBeenCalledWith(
+ expect.objectContaining({
+ exchangeRequestIds: expect.objectContaining({
+ prefix: "job-1:tools:raw",
+ }),
+ }),
+ logger,
+ );
+ expect(mocks.discoverTools).toHaveBeenCalledWith(
+ expect.objectContaining({
+ exchangeRequestIds: expect.objectContaining({
+ prefix: "job-1:tools:rewrite",
+ }),
+ }),
+ logger,
+ );
+ expect(impressions).toHaveBeenCalledWith("Search tool impression", {
+ event: "search_tool_impression",
+ searchId: "job-1",
+ teamId: "team-1",
+ servedAt: expect.stringMatching(/^\d{4}-\d{2}-\d{2}T/),
+ auto: false,
+ fusion: { enabled: true, ran: true },
+ rewrite: "arrived",
+ tools: [
+ { id: "c/data/get", rank: 1, source: "both" },
+ { id: "a/data/get", rank: 2, source: "raw" },
+ { id: "x/data/get", rank: 3, source: "rewrite" },
+ { id: "b/data/get", rank: 4, source: "raw" },
+ { id: "domain/data/get", rank: 5, source: "raw" },
+ ],
+ fitRejected: { semantic: 1, domain: 1 },
+ exchangeRequestIds: {
+ raw: ["job-1:tools:raw:1"],
+ rewrite: ["job-1:tools:rewrite:1"],
+ },
+ });
+ expect(JSON.stringify(impressions.mock.calls)).not.toContain(
+ "who owns apple stock",
+ );
+ expect(logger.info).not.toHaveBeenCalledWith(
+ "Search tool impression",
+ expect.anything(),
+ );
+ });
+
+ it("reports no fusion when the rewrite found no tools", async () => {
+ mocks.inferToolIntent.mockResolvedValue(intent);
+ mocks.discoverTools.mockImplementation(
+ async (input: { queries?: string[] }) =>
+ input.queries?.[0] === "who owns apple stock"
+ ? { items: [tool("a")], semanticMatches: 1 }
+ : { items: [], semanticMatches: 0 },
+ );
+ const result = await fused("who owns apple stock");
+ expect(result.toolFusion).toBe(false);
+ expect(impressions).toHaveBeenCalledWith(
+ "Search tool impression",
+ expect.objectContaining({
+ fusion: { enabled: true, ran: false },
+ rewrite: "empty",
+ }),
+ );
+ });
+
it("drops rewrite-only tools the fit check rejects", async () => {
mocks.inferToolIntent.mockResolvedValue(intent);
mocks.discoverTools.mockImplementation(
@@ -450,10 +549,18 @@ describe("executeSearch tool intent", () => {
it("waits up to the configured cap past the raw lookup for rewrite tools", async () => {
mocks.inferToolIntent.mockResolvedValue(intent);
- mocks.discoverTools.mockImplementation((input: { queries?: string[] }) =>
- input.queries?.[0] === "who owns apple stock"
- ? Promise.resolve({ items: [tool("a")], semanticMatches: 1 })
- : new Promise(() => {}),
+ mocks.discoverTools.mockImplementation(
+ (input: {
+ queries?: string[];
+ exchangeRequestIds?: { prefix: string; sent: string[] };
+ }) => {
+ input.exchangeRequestIds?.sent.push(
+ `${input.exchangeRequestIds.prefix}:1`,
+ );
+ if (input.queries?.[0] === "who owns apple stock")
+ return Promise.resolve({ items: [tool("a")], semanticMatches: 1 });
+ return new Promise(() => {});
+ },
);
vi.useFakeTimers();
try {
@@ -466,6 +573,19 @@ describe("executeSearch tool intent", () => {
await vi.advanceTimersByTimeAsync(1);
const result = await pending;
expect(result.response.tools).toEqual([tool("a")]);
+ expect(result.toolFusion).toBe(false);
+ expect(impressions).toHaveBeenCalledWith(
+ "Search tool impression",
+ expect.objectContaining({
+ fusion: { enabled: true, ran: false },
+ rewrite: "late",
+ tools: [{ id: "a/data/get", rank: 1, source: "raw" }],
+ exchangeRequestIds: {
+ raw: ["job-1:tools:raw:1"],
+ rewrite: ["job-1:tools:rewrite:1"],
+ },
+ }),
+ );
} finally {
vi.useRealTimers();
}
@@ -481,6 +601,10 @@ describe("executeSearch tool intent", () => {
);
expect(mocks.inferToolIntent).not.toHaveBeenCalled();
expect(mocks.discoverTools).not.toHaveBeenCalled();
+ expect(impressions).not.toHaveBeenCalledWith(
+ "Search tool impression",
+ expect.anything(),
+ );
});
});
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 36723cd29..882088719 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -115,6 +115,8 @@ interface SearchExecuteResult {
/** The tools were added automatically, not requested. */
toolsAuto?: boolean;
toolsUnavailable?: boolean;
+ /** Raw-query and rewrite tools were ranked together. */
+ toolFusion: boolean;
/** Skill-card lookup returned the Exchange envelope, including an empty one. */
skillsDiscovered?: boolean;
response: SearchV2Response;
@@ -231,6 +233,10 @@ export async function executeSearch(
? inferToolIntent(query, logger)
: null;
const fuseTools = wantsTools && options.toolFusion === true;
+ const toolRequestIds = {
+ raw: { prefix: `${context.jobId}:tools:raw`, sent: [] as string[] },
+ rewrite: { prefix: `${context.jobId}:tools:rewrite`, sent: [] as string[] },
+ };
// Tools found only through the rewrite or inferred domains are added after Jev confirms they fit the query.
const extraDiscovery = toolIntent
? toolIntent
@@ -251,6 +257,7 @@ export async function executeSearch(
? []
: intent.domains.map(domain => `https://${domain}/`),
timeoutMs: Math.min(options.timeout, 3000),
+ exchangeRequestIds: toolRequestIds.rewrite,
},
logger,
);
@@ -300,6 +307,7 @@ export async function executeSearch(
resolveToolUrls = resolve;
}),
timeoutMs: options.timeout,
+ exchangeRequestIds: toolRequestIds.raw,
},
logger,
)
@@ -504,6 +512,9 @@ export async function executeSearch(
let toolsWarning: string | undefined;
let toolsUnavailable = false;
let toolsAuto = false;
+ let rewriteLate = false;
+ let rawTools: Awaited<typeof baseDiscovery> = null;
+ let extra: Awaited<typeof extraDiscovery> = null;
const waitForExtra = async (alongside?: Promise<unknown>) => {
if (!extraDiscovery) return null;
@@ -521,8 +532,15 @@ export async function executeSearch(
delay(config.ALEXANDRIA_TOOL_FUSION_WAIT_MS),
)
: Promise.all([delay(AUTO_TOOLS_WAIT_MS), alongside]);
+ const late = Symbol("late");
try {
- return await Promise.race([extraDiscovery, deadline.then(() => null)]);
+ const found = await Promise.race([
+ extraDiscovery,
+ deadline.then((): typeof late => late),
+ ]);
+ if (found !== late) return found;
+ rewriteLate = true;
+ return null;
} finally {
settled = true;
clearTimeout(timer);
@@ -537,10 +555,12 @@ export async function executeSearch(
...indexResults,
].flatMap(item => (item.url ? [item.url] : [])),
);
- const [discovery, extra] = await Promise.all([
+ const [discovery, fetched] = await Promise.all([
baseDiscovery,
waitForExtra(baseDiscovery),
]);
+ rawTools = discovery;
+ extra = fetched;
const merged = new Map<string, DiscoveredTool>();
if (fuseTools && extra) {
// Semantic matches are fused; domain-only matches follow, as they do unfused.
@@ -577,7 +597,7 @@ export async function executeSearch(
.join(" ") || undefined;
toolsUnavailable = discovery.unavailable === true && !extra?.items.length;
} else if (extraDiscovery) {
- const extra = await waitForExtra();
+ extra = await waitForExtra();
if (extra?.items.length) {
searchResponse.tools = extra.items;
toolsAuto = true;
@@ -587,6 +607,55 @@ export async function executeSearch(
}
}
+ const rewriteFound = (extra?.found.length ?? 0) > 0;
+ const toolFusionRan = fuseTools && baseDiscovery !== null && rewriteFound;
+ if (searchResponse.tools?.length) {
+ const rawKeys = new Set(rawTools?.items.map(toolKey));
+ const fitKeys = new Set(extra?.items.map(toolKey));
+ const rejected = (extra?.found ?? []).filter(
+ tool => !fitKeys.has(toolKey(tool)) && !rawKeys.has(toolKey(tool)),
+ );
+ const semanticKeys = new Set(
+ extra?.found.slice(0, extra.semanticMatches).map(toolKey),
+ );
+ // Root logger: the request logger carries the query, and this line records shape only.
+ rootLogger.info("Search tool impression", {
+ event: "search_tool_impression",
+ searchId: context.jobId,
+ teamId,
+ servedAt: new Date().toISOString(),
+ auto: toolsAuto,
+ fusion: { enabled: fuseTools, ran: toolFusionRan },
+ rewrite: !extraDiscovery
+ ? "off"
+ : rewriteFound
+ ? "arrived"
+ : rewriteLate
+ ? "late"
+ : "empty",
+ tools: searchResponse.tools.map((tool, index) => {
+ const key = toolKey(tool);
+ const raw = rawKeys.has(key);
+ const rewrite = fitKeys.has(key);
+ return {
+ id: key,
+ rank: index + 1,
+ source: raw && rewrite ? "both" : raw ? "raw" : "rewrite",
+ };
+ }),
+ fitRejected: {
+ semantic: rejected.filter(tool => semanticKeys.has(toolKey(tool)))
+ .length,
+ domain: rejected.filter(tool => !semanticKeys.has(toolKey(tool)))
+ .length,
+ },
+ exchangeRequestIds: {
+ raw: [...toolRequestIds.raw.sent],
+ rewrite: [...toolRequestIds.rewrite.sent],
+ },
+ });
+ }
+
const isZDR = options.enterprise?.includes("zdr");
const creditsPerTenResults = isZDR ? 10 : 2;
// Gov index results are free; developer index results bill like web results.
@@ -757,6 +826,7 @@ export async function executeSearch(
toolsWarning,
toolsUnavailable,
toolsAuto,
+ toolFusion: toolFusionRan,
skillsDiscovered,
totalResultsCount,
// Counted from the final response — after scraping and highlights — so it