Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(search): log the served tool list and fusion outcome per /v2/search (#5641)
Rakshith Ramprakashc9697e6b426b
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 0327c00fb..5e898a606 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -716,6 +716,7 @@ async function searchControllerInner(markAlexandriaActivity(req.auth.team_id);}if (result.response.tools) {+ setSpanAttributes(span, { "search.tool_fusion": result.toolFusion });recordDiscoveredTools(req.auth.team_id,result.toolsAuto ? "auto_search" : "search",diff --git a/apps/api/src/search/alexandria.test.ts b/apps/api/src/search/alexandria.test.tsindex ababcfe90..a67aa5509 100644--- a/apps/api/src/search/alexandria.test.ts+++ b/apps/api/src/search/alexandria.test.ts@@ -283,3 +283,35 @@ it("reports no hidden tools when the Exchange hid nothing", async () => {expect(result.items).toHaveLength(1);expect(result).not.toHaveProperty("usOnlyExcluded");});++it("numbers each lookup's Exchange request id from the caller's prefix", async () => {+ request.mockResolvedValue({+ status: 200,+ body: { success: true, creditsCost: 0, data: { items: [tool] } },+ });+ const exchangeRequestIds = {+ prefix: "search-1:tools:raw",+ sent: [] as string[],+ };+ await discoverTools(+ { ...input, queries: ["first", "second"], exchangeRequestIds },+ logger,+ );+ expect(request.mock.calls.map(([args]) => args.requestId).sort()).toEqual([+ "search-1:tools:raw:1",+ "search-1:tools:raw:2",+ ]);+ expect(exchangeRequestIds.sent.sort()).toEqual([+ "search-1:tools:raw:1",+ "search-1:tools:raw:2",+ ]);+});++it("sends no request id when the caller passes none", async () => {+ request.mockResolvedValue({+ status: 200,+ body: { success: true, creditsCost: 0, data: { items: [tool] } },+ });+ await discoverTools(input, logger);+ expect(request.mock.calls[0][0].requestId).toBeUndefined();+});diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.tsindex 09babbdba..128a1b400 100644--- a/apps/api/src/search/alexandria.ts+++ b/apps/api/src/search/alexandria.ts@@ -87,6 +87,8 @@ export async function discoverTools(connectors?: boolean;/** The team may only be served by US-based providers. */usOnly?: boolean;+ /** Names each find-tools lookup `<prefix>:<n>` and appends the id to `sent` as it goes out. */+ exchangeRequestIds?: { prefix: string; sent: string[] };},logger: Logger,): Promise<ToolDiscovery> {@@ -102,10 +104,15 @@ export async function discoverTools(return deadline - Date.now();};const lookup = async (options: Record<string, unknown>, deadline: number) => {+ const timeoutMs = remaining(deadline);+ const ids = input.exchangeRequestIds;+ const requestId = ids ? `${ids.prefix}:${ids.sent.length + 1}` : undefined;+ if (requestId) ids?.sent.push(requestId);const result = await exchangeRequest({teamId: input.teamId,path: "/v1/retrieve",- timeoutMs: remaining(deadline),+ requestId,+ timeoutMs,maximumCredits: 0,connectors: input.connectors === true,usOnly: input.usOnly === true,diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsindex af26e700e..c66feea59 100644--- a/apps/api/src/search/execute.test.ts+++ b/apps/api/src/search/execute.test.ts@@ -76,6 +76,11 @@ vi.mock("./safe-search", () => ({import { executeSearch } from "./execute";import { searchRequestSchema } from "../controllers/v2/types";+import { logger as rootLogger } from "../lib/logger";++const impressions = vi+ .spyOn(rootLogger, "info")+ .mockImplementation(() => rootLogger);const logger = {info: vi.fn(),@@ -413,6 +418,100 @@ describe("executeSearch tool intent", () => {);});+ it("logs the served tools with their rank, source and fit rejections", async () => {+ mocks.inferToolIntent.mockResolvedValue(intent);+ mocks.discoverTools.mockImplementation(+ async (input: {+ queries?: string[];+ exchangeRequestIds?: { prefix: string; sent: string[] };+ }) => {+ input.exchangeRequestIds?.sent.push(+ `${input.exchangeRequestIds.prefix}:1`,+ );+ return input.queries?.[0] === "who owns apple stock"+ ? {+ items: [tool("a"), tool("b"), tool("c"), tool("domain")],+ semanticMatches: 3,+ }+ : {+ items: [tool("x"), tool("c"), tool("y"), tool("site")],+ semanticMatches: 3,+ };+ },+ );+ mocks.verifyTools.mockImplementation(async (_query, items) =>+ items.filter(+ (item: { provider: string }) =>+ item.provider !== "y" && item.provider !== "site",+ ),+ );+ const result = await fused("who owns apple stock");+ expect(result.toolFusion).toBe(true);+ expect(mocks.discoverTools).toHaveBeenCalledWith(+ expect.objectContaining({+ exchangeRequestIds: expect.objectContaining({+ prefix: "job-1:tools:raw",+ }),+ }),+ logger,+ );+ expect(mocks.discoverTools).toHaveBeenCalledWith(+ expect.objectContaining({+ exchangeRequestIds: expect.objectContaining({+ prefix: "job-1:tools:rewrite",+ }),+ }),+ logger,+ );+ expect(impressions).toHaveBeenCalledWith("Search tool impression", {+ event: "search_tool_impression",+ searchId: "job-1",+ teamId: "team-1",+ servedAt: expect.stringMatching(/^\d{4}-\d{2}-\d{2}T/),+ auto: false,+ fusion: { enabled: true, ran: true },+ rewrite: "arrived",+ tools: [+ { id: "c/data/get", rank: 1, source: "both" },+ { id: "a/data/get", rank: 2, source: "raw" },+ { id: "x/data/get", rank: 3, source: "rewrite" },+ { id: "b/data/get", rank: 4, source: "raw" },+ { id: "domain/data/get", rank: 5, source: "raw" },+ ],+ fitRejected: { semantic: 1, domain: 1 },+ exchangeRequestIds: {+ raw: ["job-1:tools:raw:1"],+ rewrite: ["job-1:tools:rewrite:1"],+ },+ });+ expect(JSON.stringify(impressions.mock.calls)).not.toContain(+ "who owns apple stock",+ );+ expect(logger.info).not.toHaveBeenCalledWith(+ "Search tool impression",+ expect.anything(),+ );+ });++ it("reports no fusion when the rewrite found no tools", async () => {+ mocks.inferToolIntent.mockResolvedValue(intent);+ mocks.discoverTools.mockImplementation(+ async (input: { queries?: string[] }) =>+ input.queries?.[0] === "who owns apple stock"+ ? { items: [tool("a")], semanticMatches: 1 }+ : { items: [], semanticMatches: 0 },+ );+ const result = await fused("who owns apple stock");+ expect(result.toolFusion).toBe(false);+ expect(impressions).toHaveBeenCalledWith(+ "Search tool impression",+ expect.objectContaining({+ fusion: { enabled: true, ran: false },+ rewrite: "empty",+ }),+ );+ });+it("drops rewrite-only tools the fit check rejects", async () => {mocks.inferToolIntent.mockResolvedValue(intent);mocks.discoverTools.mockImplementation(@@ -450,10 +549,18 @@ describe("executeSearch tool intent", () => {it("waits up to the configured cap past the raw lookup for rewrite tools", async () => {mocks.inferToolIntent.mockResolvedValue(intent);- mocks.discoverTools.mockImplementation((input: { queries?: string[] }) =>- input.queries?.[0] === "who owns apple stock"- ? Promise.resolve({ items: [tool("a")], semanticMatches: 1 })- : new Promise(() => {}),+ mocks.discoverTools.mockImplementation(+ (input: {+ queries?: string[];+ exchangeRequestIds?: { prefix: string; sent: string[] };+ }) => {+ input.exchangeRequestIds?.sent.push(+ `${input.exchangeRequestIds.prefix}:1`,+ );+ if (input.queries?.[0] === "who owns apple stock")+ return Promise.resolve({ items: [tool("a")], semanticMatches: 1 });+ return new Promise(() => {});+ },);vi.useFakeTimers();try {@@ -466,6 +573,19 @@ describe("executeSearch tool intent", () => {await vi.advanceTimersByTimeAsync(1);const result = await pending;expect(result.response.tools).toEqual([tool("a")]);+ expect(result.toolFusion).toBe(false);+ expect(impressions).toHaveBeenCalledWith(+ "Search tool impression",+ expect.objectContaining({+ fusion: { enabled: true, ran: false },+ rewrite: "late",+ tools: [{ id: "a/data/get", rank: 1, source: "raw" }],+ exchangeRequestIds: {+ raw: ["job-1:tools:raw:1"],+ rewrite: ["job-1:tools:rewrite:1"],+ },+ }),+ );} finally {vi.useRealTimers();}@@ -481,6 +601,10 @@ describe("executeSearch tool intent", () => {);expect(mocks.inferToolIntent).not.toHaveBeenCalled();expect(mocks.discoverTools).not.toHaveBeenCalled();+ expect(impressions).not.toHaveBeenCalledWith(+ "Search tool impression",+ expect.anything(),+ );});});diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 36723cd29..882088719 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -115,6 +115,8 @@ interface SearchExecuteResult {/** The tools were added automatically, not requested. */toolsAuto?: boolean;toolsUnavailable?: boolean;+ /** Raw-query and rewrite tools were ranked together. */+ toolFusion: boolean;/** Skill-card lookup returned the Exchange envelope, including an empty one. */skillsDiscovered?: boolean;response: SearchV2Response;@@ -231,6 +233,10 @@ export async function executeSearch(? inferToolIntent(query, logger): null;const fuseTools = wantsTools && options.toolFusion === true;+ const toolRequestIds = {+ raw: { prefix: `${context.jobId}:tools:raw`, sent: [] as string[] },+ rewrite: { prefix: `${context.jobId}:tools:rewrite`, sent: [] as string[] },+ };// Tools found only through the rewrite or inferred domains are added after Jev confirms they fit the query.const extraDiscovery = toolIntent? toolIntent@@ -251,6 +257,7 @@ export async function executeSearch(? []: intent.domains.map(domain => `https://${domain}/`),timeoutMs: Math.min(options.timeout, 3000),+ exchangeRequestIds: toolRequestIds.rewrite,},logger,);@@ -300,6 +307,7 @@ export async function executeSearch(resolveToolUrls = resolve;}),timeoutMs: options.timeout,+ exchangeRequestIds: toolRequestIds.raw,},logger,)@@ -504,6 +512,9 @@ export async function executeSearch(let toolsWarning: string | undefined;let toolsUnavailable = false;let toolsAuto = false;+ let rewriteLate = false;+ let rawTools: Awaited<typeof baseDiscovery> = null;+ let extra: Awaited<typeof extraDiscovery> = null;const waitForExtra = async (alongside?: Promise<unknown>) => {if (!extraDiscovery) return null;@@ -521,8 +532,15 @@ export async function executeSearch(delay(config.ALEXANDRIA_TOOL_FUSION_WAIT_MS),): Promise.all([delay(AUTO_TOOLS_WAIT_MS), alongside]);+ const late = Symbol("late");try {- return await Promise.race([extraDiscovery, deadline.then(() => null)]);+ const found = await Promise.race([+ extraDiscovery,+ deadline.then((): typeof late => late),+ ]);+ if (found !== late) return found;+ rewriteLate = true;+ return null;} finally {settled = true;clearTimeout(timer);@@ -537,10 +555,12 @@ export async function executeSearch(...indexResults,].flatMap(item => (item.url ? [item.url] : [])),);- const [discovery, extra] = await Promise.all([+ const [discovery, fetched] = await Promise.all([baseDiscovery,waitForExtra(baseDiscovery),]);+ rawTools = discovery;+ extra = fetched;const merged = new Map<string, DiscoveredTool>();if (fuseTools && extra) {// Semantic matches are fused; domain-only matches follow, as they do unfused.@@ -577,7 +597,7 @@ export async function executeSearch(.join(" ") || undefined;toolsUnavailable = discovery.unavailable === true && !extra?.items.length;} else if (extraDiscovery) {- const extra = await waitForExtra();+ extra = await waitForExtra();if (extra?.items.length) {searchResponse.tools = extra.items;toolsAuto = true;@@ -587,6 +607,55 @@ export async function executeSearch(}}+ const rewriteFound = (extra?.found.length ?? 0) > 0;+ const toolFusionRan = fuseTools && baseDiscovery !== null && rewriteFound;+ if (searchResponse.tools?.length) {+ const rawKeys = new Set(rawTools?.items.map(toolKey));+ const fitKeys = new Set(extra?.items.map(toolKey));+ const rejected = (extra?.found ?? []).filter(+ tool => !fitKeys.has(toolKey(tool)) && !rawKeys.has(toolKey(tool)),+ );+ const semanticKeys = new Set(+ extra?.found.slice(0, extra.semanticMatches).map(toolKey),+ );+ // Root logger: the request logger carries the query, and this line records shape only.+ rootLogger.info("Search tool impression", {+ event: "search_tool_impression",+ searchId: context.jobId,+ teamId,+ servedAt: new Date().toISOString(),+ auto: toolsAuto,+ fusion: { enabled: fuseTools, ran: toolFusionRan },+ rewrite: !extraDiscovery+ ? "off"+ : rewriteFound+ ? "arrived"+ : rewriteLate+ ? "late"+ : "empty",+ tools: searchResponse.tools.map((tool, index) => {+ const key = toolKey(tool);+ const raw = rawKeys.has(key);+ const rewrite = fitKeys.has(key);+ return {+ id: key,+ rank: index + 1,+ source: raw && rewrite ? "both" : raw ? "raw" : "rewrite",+ };+ }),+ fitRejected: {+ semantic: rejected.filter(tool => semanticKeys.has(toolKey(tool)))+ .length,+ domain: rejected.filter(tool => !semanticKeys.has(toolKey(tool)))+ .length,+ },+ exchangeRequestIds: {+ raw: [...toolRequestIds.raw.sent],+ rewrite: [...toolRequestIds.rewrite.sent],+ },+ });+ }+const isZDR = options.enterprise?.includes("zdr");const creditsPerTenResults = isZDR ? 10 : 2;// Gov index results are free; developer index results bill like web results.@@ -757,6 +826,7 @@ export async function executeSearch(toolsWarning,toolsUnavailable,toolsAuto,+ toolFusion: toolFusionRan,skillsDiscovered,totalResultsCount,// Counted from the final response — after scraping and highlights — so it