Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(search): fuse rewrite tool discovery by reciprocal rank (flagged off) (#5595)
Rakshith Ramprakashdb23872cf8ff
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex e986006be..657f4a16c 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -314,6 +314,7 @@ jobs:pnpm exec vitest runsrc/search/safe-search.test.tssrc/search/execute.test.ts+ src/search/tool-intent.test.ts--reporter=defaultworking-directory: apps/apidiff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 935f844d4..e4716bab9 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -436,6 +436,16 @@ const configSchema = z.object({// team flag).ANTHROPIC_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),ALEXANDRIA_AUTO_TOOLS: z.stringbool().default(false),+ // Rank rewrite-found tools together with raw-query tools by reciprocal rank+ // fusion (also the alexandriaToolFusion team flag), waiting up to+ // ALEXANDRIA_TOOL_FUSION_WAIT_MS past the raw lookup for them.+ ALEXANDRIA_TOOL_FUSION: z.stringbool().default(false),+ ALEXANDRIA_TOOL_FUSION_WAIT_MS: z.coerce+ .number()+ .int()+ .min(0)+ .max(5000)+ .default(1000),// Exchange (routed data sources service)FIRE_EXCHANGE_URL: z.url().optional(),diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex aeb519786..0327c00fb 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -69,6 +69,7 @@ import { recordDiscoveredTools } from "../../lib/alexandria-discovery";import {autoToolsEnabled,queryInferenceEnabled,+ toolFusionEnabled,} from "../../search/tool-intent";import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";import { UsOnlyUpstreamRefusalError } from "../../lib/research-upstream";@@ -556,6 +557,7 @@ async function searchControllerInner(domainTools: req.body.domainTools,toolDetail: req.body.toolDetail,inferToolQuery: queryInferenceEnabled(),+ toolFusion: toolFusionEnabled(req.acuc?.flags),autoTools:(sourcesOmitted || req.body.domainTools === true) &&!req.auth.team_id.startsWith("preview_keyless_") &&diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex e93d0a8b3..9474f38b1 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -2138,6 +2138,7 @@ export const LOCKDOWN_DEFAULT_MAX_AGE_MS = 2 * 365 * 24 * 60 * 60 * 1000;export type TeamFlags = {exchangeRetrieve?: boolean;alexandriaAutoTools?: boolean;+ alexandriaToolFusion?: boolean;ignoreRobots?: "disabled" | "allowed" | "forced";customRobotsAgent?: "disabled" | "allowed";threatProtection?: "disabled" | "allowed" | "forced";diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.tsindex a7630c9f9..09babbdba 100644--- a/apps/api/src/search/alexandria.ts+++ b/apps/api/src/search/alexandria.ts@@ -18,6 +18,8 @@ type ToolDiscovery = {usOnlyExcluded?: number;/** Every discovery leg failed and nothing was found. */unavailable?: true;+ /** How many leading items the semantic lookup found; the rest matched by domain only. */+ semanticMatches: number;};type SourceLike = string | { type: string };@@ -311,6 +313,7 @@ export async function discoverTools(})();const [semanticTools, domainTools] = await Promise.all([semantic, domain]);for (const tool of semanticTools ?? []) merge(tool, "semantic");+ const semanticMatches = items.size;for (const { tool, urls } of domainTools ?? []) merge(tool, "domain", urls);return {items: [...items.values()].map(tool =>@@ -322,6 +325,7 @@ export async function discoverTools(}: tool,),+ semanticMatches,...(answered < attempted? { warning: "Some tool discovery results are unavailable." }: {}),diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsindex 1947a39e4..af26e700e 100644--- a/apps/api/src/search/execute.test.ts+++ b/apps/api/src/search/execute.test.ts@@ -12,6 +12,7 @@ const mocks = vi.hoisted(() => ({vi.mock("./tool-intent", () => ({AUTO_TOOLS_WAIT_MS: 300,+ FUSED_FIT_THRESHOLD: 0.2,inferToolIntent: mocks.inferToolIntent,verifyTools: mocks.verifyTools,}));@@ -355,6 +356,122 @@ describe("executeSearch tool intent", () => {}});+ describe("with tool fusion", () => {+ const tool = (provider: string) => ({ provider, capability: "data/get" });+ const intent = {+ toolQueries: ["institutional holders of a public company"],+ domains: [],+ needsTools: true,+ };+ const fused = (query: string) =>+ executeSearch(+ {+ ...options([]),+ query,+ sources: [{ type: "alexandria" }],+ inferToolQuery: true,+ toolFusion: true,+ },+ context,+ logger,+ );++ it("ranks rewrite tools among raw-query tools by reciprocal rank", async () => {+ mocks.inferToolIntent.mockResolvedValue(intent);+ mocks.discoverTools.mockImplementation(+ async (input: { queries?: string[] }) =>+ input.queries?.[0] === "who owns apple stock"+ ? {+ items: [tool("a"), tool("b"), tool("c"), tool("domain")],+ semanticMatches: 3,+ }+ : {+ items: [tool("x"), tool("c"), tool("y"), tool("site")],+ semanticMatches: 3,+ },+ );+ const result = await fused("who owns apple stock");+ expect(result.response.tools?.map(found => found.provider)).toEqual([+ "c",+ "a",+ "x",+ "b",+ "y",+ "domain",+ "site",+ ]);+ expect(mocks.verifyTools).toHaveBeenCalledWith(+ "who owns apple stock",+ [tool("x"), tool("c"), tool("y")],+ logger,+ 0.2,+ );+ expect(mocks.verifyTools).toHaveBeenCalledWith(+ "who owns apple stock",+ [tool("site")],+ logger,+ );+ });++ it("drops rewrite-only tools the fit check rejects", async () => {+ mocks.inferToolIntent.mockResolvedValue(intent);+ mocks.discoverTools.mockImplementation(+ async (input: { queries?: string[] }) =>+ input.queries?.[0] === "who owns apple stock"+ ? { items: [tool("a"), tool("b")], semanticMatches: 2 }+ : { items: [tool("x"), tool("b")], semanticMatches: 2 },+ );+ mocks.verifyTools.mockResolvedValue([]);+ const result = await fused("who owns apple stock");+ expect(result.response.tools?.map(found => found.provider)).toEqual([+ "b",+ "a",+ ]);+ });++ it("scores rewrite tools by their original rank when earlier ones are rejected", async () => {+ mocks.inferToolIntent.mockResolvedValue(intent);+ mocks.discoverTools.mockImplementation(+ async (input: { queries?: string[] }) =>+ input.queries?.[0] === "who owns apple stock"+ ? { items: [tool("a"), tool("b")], semanticMatches: 2 }+ : { items: [tool("x"), tool("y")], semanticMatches: 2 },+ );+ mocks.verifyTools.mockImplementation(async (_query, items) =>+ items.filter((item: { provider: string }) => item.provider !== "x"),+ );+ const result = await fused("who owns apple stock");+ expect(result.response.tools?.map(found => found.provider)).toEqual([+ "a",+ "b",+ "y",+ ]);+ });++ it("waits up to the configured cap past the raw lookup for rewrite tools", async () => {+ mocks.inferToolIntent.mockResolvedValue(intent);+ mocks.discoverTools.mockImplementation((input: { queries?: string[] }) =>+ input.queries?.[0] === "who owns apple stock"+ ? Promise.resolve({ items: [tool("a")], semanticMatches: 1 })+ : new Promise(() => {}),+ );+ vi.useFakeTimers();+ try {+ let done = false;+ const pending = fused("who owns apple stock").finally(() => {+ done = true;+ });+ await vi.advanceTimersByTimeAsync(999);+ expect(done).toBe(false);+ await vi.advanceTimersByTimeAsync(1);+ const result = await pending;+ expect(result.response.tools).toEqual([tool("a")]);+ } finally {+ vi.useRealTimers();+ }+ });+ });+it("never adds tools under zero data retention", async () => {mocks.search.mockResolvedValue({ web: [webResult] });await executeSearch(diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 4bad67142..36723cd29 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -12,9 +12,11 @@ import {} from "./skill-cards";import {AUTO_TOOLS_WAIT_MS,+ FUSED_FIT_THRESHOLD,inferToolIntent,verifyTools,} from "./tool-intent";+import type { DiscoveredTool } from "../services/alexandria/contracts";import { search } from "./v2";import {SearchV2Response,@@ -80,6 +82,8 @@ interface SearchOptions {inferToolQuery?: boolean;/** Add compact tools to a search that did not ask for them, when the query needs them. */autoTools?: boolean;+ /** Rank rewrite-found tools together with raw-query tools instead of after them. */+ toolFusion?: boolean;timeout: number;}@@ -124,6 +128,57 @@ interface SearchExecuteResult {shouldScrape: boolean;}+const toolKey = (tool: DiscoveredTool) => `${tool.provider}/${tool.capability}`;++function addTool(tools: Map<string, DiscoveredTool>, tool: DiscoveredTool) {+ const key = toolKey(tool);+ const previous = tools.get(key);+ tools.set(+ key,+ previous && "matchedBy" in previous && "matchedBy" in tool+ ? {+ ...previous,+ matchedBy: [+ ...new Set([+ ...(previous.matchedBy ?? []),+ ...(tool.matchedBy ?? []),+ ]),+ ],+ matchedUrls: [+ ...new Set([+ ...(previous.matchedUrls ?? []),+ ...(tool.matchedUrls ?? []),+ ]),+ ],+ }+ : (previous ?? tool),+ );+}++const RRF_K = 60;++/**+ * Orders tools by reciprocal rank fusion; ties keep the earlier list's order.+ * Tools `keep` rejects are left out without moving the ranks of the rest.+ */+function fuseRankings(+ lists: DiscoveredTool[][],+ keep: (tool: DiscoveredTool) => boolean,+): DiscoveredTool[] {+ const tools = new Map<string, DiscoveredTool>();+ const scores = new Map<string, number>();+ for (const list of lists)+ list.forEach((tool, index) => {+ if (!keep(tool)) return;+ addTool(tools, tool);+ const key = toolKey(tool);+ scores.set(key, (scores.get(key) ?? 0) + 1 / (RRF_K + index + 1));+ });+ return [...tools.entries()]+ .sort(([a], [b]) => scores.get(b)! - scores.get(a)!)+ .map(([, tool]) => tool);+}+export async function executeSearch(options: SearchOptions,context: SearchContext,@@ -175,6 +230,7 @@ export async function executeSearch(query.trim()? inferToolIntent(query, logger): null;+ const fuseTools = wantsTools && options.toolFusion === true;// Tools found only through the rewrite or inferred domains are added after Jev confirms they fit the query.const extraDiscovery = toolIntent? toolIntent@@ -199,7 +255,28 @@ export async function executeSearch(logger,);return {- items: await verifyTools(query, found.items, logger),+ // Only fused semantic matches get the lower bar; domain matches keep the default.+ items: fuseTools+ ? (+ await Promise.all([+ verifyTools(+ query,+ found.items.slice(0, found.semanticMatches),+ logger,+ FUSED_FIT_THRESHOLD,+ ),+ found.items.length > found.semanticMatches+ ? verifyTools(+ query,+ found.items.slice(found.semanticMatches),+ logger,+ )+ : [],+ ])+ ).flat()+ : await verifyTools(query, found.items, logger),+ found: found.items,+ semanticMatches: found.semanticMatches,usOnlyExcluded: found.usOnlyExcluded ?? 0,};})@@ -431,16 +508,23 @@ export async function executeSearch(const waitForExtra = async (alongside?: Promise<unknown>) => {if (!extraDiscovery) return null;let timer: NodeJS.Timeout | undefined;- const deadline = new Promise<void>(resolve => {- timer = setTimeout(resolve, AUTO_TOOLS_WAIT_MS);- timer.unref();- });+ let settled = false;+ const delay = (ms: number) =>+ new Promise<void>(resolve => {+ if (settled) return resolve();+ timer = setTimeout(resolve, ms);+ timer.unref();+ });+ // Fused tools are worth waiting for past the raw lookup; appended ones are not.+ const deadline = fuseTools+ ? Promise.resolve(alongside).then(() =>+ delay(config.ALEXANDRIA_TOOL_FUSION_WAIT_MS),+ )+ : Promise.all([delay(AUTO_TOOLS_WAIT_MS), alongside]);try {- return await Promise.race([- extraDiscovery,- Promise.all([deadline, alongside]).then(() => null),- ]);+ return await Promise.race([extraDiscovery, deadline.then(() => null)]);} finally {+ settled = true;clearTimeout(timer);}};@@ -457,35 +541,26 @@ export async function executeSearch(baseDiscovery,waitForExtra(baseDiscovery),]);- const merged = new Map(- discovery.items.map(tool => [- `${tool.provider}/${tool.capability}`,- tool,- ]),- );- for (const tool of extra?.items ?? []) {- const key = `${tool.provider}/${tool.capability}`;- const previous = merged.get(key);- merged.set(- key,- previous && "matchedBy" in previous && "matchedBy" in tool- ? {- ...previous,- matchedBy: [- ...new Set([- ...(previous.matchedBy ?? []),- ...(tool.matchedBy ?? []),- ]),- ],- matchedUrls: [- ...new Set([- ...(previous.matchedUrls ?? []),- ...(tool.matchedUrls ?? []),- ]),- ],- }- : (previous ?? tool),- );+ const merged = new Map<string, DiscoveredTool>();+ if (fuseTools && extra) {+ // Semantic matches are fused; domain-only matches follow, as they do unfused.+ const fits = new Set(extra.items.map(toolKey));+ const baseSemantic = discovery.items.slice(0, discovery.semanticMatches);+ const baseKeys = new Set(baseSemantic.map(toolKey));+ for (const tool of [+ ...fuseRankings(+ [baseSemantic, extra.found.slice(0, extra.semanticMatches)],+ tool => baseKeys.has(toolKey(tool)) || fits.has(toolKey(tool)),+ ),+ ...discovery.items.slice(baseSemantic.length),+ ...extra.found+ .slice(extra.semanticMatches)+ .filter(tool => fits.has(toolKey(tool))),+ ])+ addTool(merged, tool);+ } else {+ for (const tool of [...discovery.items, ...(extra?.items ?? [])])+ addTool(merged, tool);}searchResponse.tools = [...merged.values()];// Both legs can hide the same tools, so the larger count is reported.diff --git a/apps/api/src/search/tool-intent.test.ts b/apps/api/src/search/tool-intent.test.tsindex 1a35fbdaf..6993b9e92 100644--- a/apps/api/src/search/tool-intent.test.ts+++ b/apps/api/src/search/tool-intent.test.ts@@ -10,6 +10,7 @@ const config = vi.hoisted(() => ({ANTHROPIC_API_KEY: undefined as string | undefined,TYPESAFE_API_KEY: "key" as string | undefined,ALEXANDRIA_AUTO_TOOLS: false,+ ALEXANDRIA_TOOL_FUSION: false,}));vi.mock("ai", () => ({ generateObject: mocks.generateObject }));@@ -28,6 +29,7 @@ import {inferToolIntent,parseSearchQuery,queryInferenceEnabled,+ toolFusionEnabled,verifyTools,} from "./tool-intent";@@ -124,6 +126,15 @@ describe("feature gates", () => {expect(queryInferenceEnabled()).toBe(false);config.TYPESAFE_API_KEY = "key";});++ it("keep tool fusion off unless the config or team flag turns it on", () => {+ expect(toolFusionEnabled({})).toBe(false);+ expect(toolFusionEnabled(null)).toBe(false);+ expect(toolFusionEnabled({ alexandriaToolFusion: true })).toBe(true);+ config.ALEXANDRIA_TOOL_FUSION = true;+ expect(toolFusionEnabled({})).toBe(true);+ config.ALEXANDRIA_TOOL_FUSION = false;+ });});it("sends no semantic query for an operator-only query", async () => {@@ -158,3 +169,16 @@ it("keeps only the tools Jev judges to fit, and drops unjudged ones", async () =tools[0],]);});++it("keeps tools at or above a lower fit threshold when one is given", async () => {+ const tools = [+ { provider: "fiscal-ai", capability: "ownership/holders", description: "" },+ { provider: "sec-gov", capability: "filings/filings", description: "" },+ ];+ systemOne+ .mockResolvedValueOnce({ answers: { fits: { noul: 0.2 } } })+ .mockResolvedValueOnce({ answers: { fits: { noul: 0.1 } } });+ expect(await verifyTools("who owns apple stock", tools, logger, 0.2)).toEqual(+ [tools[0]],+ );+});diff --git a/apps/api/src/search/tool-intent.ts b/apps/api/src/search/tool-intent.tsindex 5ced17a0f..beb926b5a 100644--- a/apps/api/src/search/tool-intent.ts+++ b/apps/api/src/search/tool-intent.ts@@ -37,6 +37,9 @@ export const autoToolsEnabled = (flags: TeamFlags | null | undefined) =>modelsConfigured() &&(config.ALEXANDRIA_AUTO_TOOLS || flags?.alexandriaAutoTools === true);+export const toolFusionEnabled = (flags: TeamFlags | null | undefined) =>+ config.ALEXANDRIA_TOOL_FUSION || flags?.alexandriaToolFusion === true;+const OPERATORS = new Set(["site","inurl",@@ -264,6 +267,8 @@ export async function inferToolIntent(const VERIFY_TIMEOUT_MS = 1500;/** Jev's probability that a tool fits the query, at or above which it is kept. */const FIT_THRESHOLD = 0.5;+/** Lower bar for rewrite-found tools when they are fused into the raw-query ranking. */+export const FUSED_FIT_THRESHOLD = 0.2;const fitQuestion = {fits: noul(@@ -288,6 +293,7 @@ export async function verifyTools(query: string,tools: DiscoveredTool[],logger: Logger,+ threshold = FIT_THRESHOLD,): Promise<DiscoveredTool[]> {const typesafe = getTypeSafeClient();if (!typesafe || !tools.length) return [];@@ -309,7 +315,7 @@ export async function verifyTools(},{ signal, timeout: VERIFY_TIMEOUT_MS, retry: { maxRetries: 0 } },);- return answers.fits.noul >= FIT_THRESHOLD;+ return answers.fits.noul >= threshold;} catch {failed++;return false;