Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(search): fuse rewrite tool discovery by reciprocal rank (flagged off) (#5595)

Rakshith Ramprakashdb23872cf8ff
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index e986006be..657f4a16c 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -314,6 +314,7 @@ jobs:
pnpm exec vitest run
src/search/safe-search.test.ts
src/search/execute.test.ts
+ src/search/tool-intent.test.ts
--reporter=default
working-directory: apps/api
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 935f844d4..e4716bab9 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -436,6 +436,16 @@ const configSchema = z.object({
// team flag).
ANTHROPIC_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),
ALEXANDRIA_AUTO_TOOLS: z.stringbool().default(false),
+ // Rank rewrite-found tools together with raw-query tools by reciprocal rank
+ // fusion (also the alexandriaToolFusion team flag), waiting up to
+ // ALEXANDRIA_TOOL_FUSION_WAIT_MS past the raw lookup for them.
+ ALEXANDRIA_TOOL_FUSION: z.stringbool().default(false),
+ ALEXANDRIA_TOOL_FUSION_WAIT_MS: z.coerce
+ .number()
+ .int()
+ .min(0)
+ .max(5000)
+ .default(1000),
// Exchange (routed data sources service)
FIRE_EXCHANGE_URL: z.url().optional(),
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index aeb519786..0327c00fb 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -69,6 +69,7 @@ import { recordDiscoveredTools } from "../../lib/alexandria-discovery";
import {
autoToolsEnabled,
queryInferenceEnabled,
+ toolFusionEnabled,
} from "../../search/tool-intent";
import { isUsOnly, usOnlyLocationRefusal } from "../../lib/us-only";
import { UsOnlyUpstreamRefusalError } from "../../lib/research-upstream";
@@ -556,6 +557,7 @@ async function searchControllerInner(
domainTools: req.body.domainTools,
toolDetail: req.body.toolDetail,
inferToolQuery: queryInferenceEnabled(),
+ toolFusion: toolFusionEnabled(req.acuc?.flags),
autoTools:
(sourcesOmitted || req.body.domainTools === true) &&
!req.auth.team_id.startsWith("preview_keyless_") &&
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index e93d0a8b3..9474f38b1 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -2138,6 +2138,7 @@ export const LOCKDOWN_DEFAULT_MAX_AGE_MS = 2 * 365 * 24 * 60 * 60 * 1000;
export type TeamFlags = {
exchangeRetrieve?: boolean;
alexandriaAutoTools?: boolean;
+ alexandriaToolFusion?: boolean;
ignoreRobots?: "disabled" | "allowed" | "forced";
customRobotsAgent?: "disabled" | "allowed";
threatProtection?: "disabled" | "allowed" | "forced";
diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.ts
index a7630c9f9..09babbdba 100644
--- a/apps/api/src/search/alexandria.ts
+++ b/apps/api/src/search/alexandria.ts
@@ -18,6 +18,8 @@ type ToolDiscovery = {
usOnlyExcluded?: number;
/** Every discovery leg failed and nothing was found. */
unavailable?: true;
+ /** How many leading items the semantic lookup found; the rest matched by domain only. */
+ semanticMatches: number;
};
type SourceLike = string | { type: string };
@@ -311,6 +313,7 @@ export async function discoverTools(
})();
const [semanticTools, domainTools] = await Promise.all([semantic, domain]);
for (const tool of semanticTools ?? []) merge(tool, "semantic");
+ const semanticMatches = items.size;
for (const { tool, urls } of domainTools ?? []) merge(tool, "domain", urls);
return {
items: [...items.values()].map(tool =>
@@ -322,6 +325,7 @@ export async function discoverTools(
}
: tool,
),
+ semanticMatches,
...(answered < attempted
? { warning: "Some tool discovery results are unavailable." }
: {}),
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.ts
index 1947a39e4..af26e700e 100644
--- a/apps/api/src/search/execute.test.ts
+++ b/apps/api/src/search/execute.test.ts
@@ -12,6 +12,7 @@ const mocks = vi.hoisted(() => ({
vi.mock("./tool-intent", () => ({
AUTO_TOOLS_WAIT_MS: 300,
+ FUSED_FIT_THRESHOLD: 0.2,
inferToolIntent: mocks.inferToolIntent,
verifyTools: mocks.verifyTools,
}));
@@ -355,6 +356,122 @@ describe("executeSearch tool intent", () => {
}
});
+ describe("with tool fusion", () => {
+ const tool = (provider: string) => ({ provider, capability: "data/get" });
+ const intent = {
+ toolQueries: ["institutional holders of a public company"],
+ domains: [],
+ needsTools: true,
+ };
+ const fused = (query: string) =>
+ executeSearch(
+ {
+ ...options([]),
+ query,
+ sources: [{ type: "alexandria" }],
+ inferToolQuery: true,
+ toolFusion: true,
+ },
+ context,
+ logger,
+ );
+
+ it("ranks rewrite tools among raw-query tools by reciprocal rank", async () => {
+ mocks.inferToolIntent.mockResolvedValue(intent);
+ mocks.discoverTools.mockImplementation(
+ async (input: { queries?: string[] }) =>
+ input.queries?.[0] === "who owns apple stock"
+ ? {
+ items: [tool("a"), tool("b"), tool("c"), tool("domain")],
+ semanticMatches: 3,
+ }
+ : {
+ items: [tool("x"), tool("c"), tool("y"), tool("site")],
+ semanticMatches: 3,
+ },
+ );
+ const result = await fused("who owns apple stock");
+ expect(result.response.tools?.map(found => found.provider)).toEqual([
+ "c",
+ "a",
+ "x",
+ "b",
+ "y",
+ "domain",
+ "site",
+ ]);
+ expect(mocks.verifyTools).toHaveBeenCalledWith(
+ "who owns apple stock",
+ [tool("x"), tool("c"), tool("y")],
+ logger,
+ 0.2,
+ );
+ expect(mocks.verifyTools).toHaveBeenCalledWith(
+ "who owns apple stock",
+ [tool("site")],
+ logger,
+ );
+ });
+
+ it("drops rewrite-only tools the fit check rejects", async () => {
+ mocks.inferToolIntent.mockResolvedValue(intent);
+ mocks.discoverTools.mockImplementation(
+ async (input: { queries?: string[] }) =>
+ input.queries?.[0] === "who owns apple stock"
+ ? { items: [tool("a"), tool("b")], semanticMatches: 2 }
+ : { items: [tool("x"), tool("b")], semanticMatches: 2 },
+ );
+ mocks.verifyTools.mockResolvedValue([]);
+ const result = await fused("who owns apple stock");
+ expect(result.response.tools?.map(found => found.provider)).toEqual([
+ "b",
+ "a",
+ ]);
+ });
+
+ it("scores rewrite tools by their original rank when earlier ones are rejected", async () => {
+ mocks.inferToolIntent.mockResolvedValue(intent);
+ mocks.discoverTools.mockImplementation(
+ async (input: { queries?: string[] }) =>
+ input.queries?.[0] === "who owns apple stock"
+ ? { items: [tool("a"), tool("b")], semanticMatches: 2 }
+ : { items: [tool("x"), tool("y")], semanticMatches: 2 },
+ );
+ mocks.verifyTools.mockImplementation(async (_query, items) =>
+ items.filter((item: { provider: string }) => item.provider !== "x"),
+ );
+ const result = await fused("who owns apple stock");
+ expect(result.response.tools?.map(found => found.provider)).toEqual([
+ "a",
+ "b",
+ "y",
+ ]);
+ });
+
+ it("waits up to the configured cap past the raw lookup for rewrite tools", async () => {
+ mocks.inferToolIntent.mockResolvedValue(intent);
+ mocks.discoverTools.mockImplementation((input: { queries?: string[] }) =>
+ input.queries?.[0] === "who owns apple stock"
+ ? Promise.resolve({ items: [tool("a")], semanticMatches: 1 })
+ : new Promise(() => {}),
+ );
+ vi.useFakeTimers();
+ try {
+ let done = false;
+ const pending = fused("who owns apple stock").finally(() => {
+ done = true;
+ });
+ await vi.advanceTimersByTimeAsync(999);
+ expect(done).toBe(false);
+ await vi.advanceTimersByTimeAsync(1);
+ const result = await pending;
+ expect(result.response.tools).toEqual([tool("a")]);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+ });
+
it("never adds tools under zero data retention", async () => {
mocks.search.mockResolvedValue({ web: [webResult] });
await executeSearch(
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 4bad67142..36723cd29 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -12,9 +12,11 @@ import {
} from "./skill-cards";
import {
AUTO_TOOLS_WAIT_MS,
+ FUSED_FIT_THRESHOLD,
inferToolIntent,
verifyTools,
} from "./tool-intent";
+import type { DiscoveredTool } from "../services/alexandria/contracts";
import { search } from "./v2";
import {
SearchV2Response,
@@ -80,6 +82,8 @@ interface SearchOptions {
inferToolQuery?: boolean;
/** Add compact tools to a search that did not ask for them, when the query needs them. */
autoTools?: boolean;
+ /** Rank rewrite-found tools together with raw-query tools instead of after them. */
+ toolFusion?: boolean;
timeout: number;
}
@@ -124,6 +128,57 @@ interface SearchExecuteResult {
shouldScrape: boolean;
}
+const toolKey = (tool: DiscoveredTool) => `${tool.provider}/${tool.capability}`;
+
+function addTool(tools: Map<string, DiscoveredTool>, tool: DiscoveredTool) {
+ const key = toolKey(tool);
+ const previous = tools.get(key);
+ tools.set(
+ key,
+ previous && "matchedBy" in previous && "matchedBy" in tool
+ ? {
+ ...previous,
+ matchedBy: [
+ ...new Set([
+ ...(previous.matchedBy ?? []),
+ ...(tool.matchedBy ?? []),
+ ]),
+ ],
+ matchedUrls: [
+ ...new Set([
+ ...(previous.matchedUrls ?? []),
+ ...(tool.matchedUrls ?? []),
+ ]),
+ ],
+ }
+ : (previous ?? tool),
+ );
+}
+
+const RRF_K = 60;
+
+/**
+ * Orders tools by reciprocal rank fusion; ties keep the earlier list's order.
+ * Tools `keep` rejects are left out without moving the ranks of the rest.
+ */
+function fuseRankings(
+ lists: DiscoveredTool[][],
+ keep: (tool: DiscoveredTool) => boolean,
+): DiscoveredTool[] {
+ const tools = new Map<string, DiscoveredTool>();
+ const scores = new Map<string, number>();
+ for (const list of lists)
+ list.forEach((tool, index) => {
+ if (!keep(tool)) return;
+ addTool(tools, tool);
+ const key = toolKey(tool);
+ scores.set(key, (scores.get(key) ?? 0) + 1 / (RRF_K + index + 1));
+ });
+ return [...tools.entries()]
+ .sort(([a], [b]) => scores.get(b)! - scores.get(a)!)
+ .map(([, tool]) => tool);
+}
+
export async function executeSearch(
options: SearchOptions,
context: SearchContext,
@@ -175,6 +230,7 @@ export async function executeSearch(
query.trim()
? inferToolIntent(query, logger)
: null;
+ const fuseTools = wantsTools && options.toolFusion === true;
// Tools found only through the rewrite or inferred domains are added after Jev confirms they fit the query.
const extraDiscovery = toolIntent
? toolIntent
@@ -199,7 +255,28 @@ export async function executeSearch(
logger,
);
return {
- items: await verifyTools(query, found.items, logger),
+ // Only fused semantic matches get the lower bar; domain matches keep the default.
+ items: fuseTools
+ ? (
+ await Promise.all([
+ verifyTools(
+ query,
+ found.items.slice(0, found.semanticMatches),
+ logger,
+ FUSED_FIT_THRESHOLD,
+ ),
+ found.items.length > found.semanticMatches
+ ? verifyTools(
+ query,
+ found.items.slice(found.semanticMatches),
+ logger,
+ )
+ : [],
+ ])
+ ).flat()
+ : await verifyTools(query, found.items, logger),
+ found: found.items,
+ semanticMatches: found.semanticMatches,
usOnlyExcluded: found.usOnlyExcluded ?? 0,
};
})
@@ -431,16 +508,23 @@ export async function executeSearch(
const waitForExtra = async (alongside?: Promise<unknown>) => {
if (!extraDiscovery) return null;
let timer: NodeJS.Timeout | undefined;
- const deadline = new Promise<void>(resolve => {
- timer = setTimeout(resolve, AUTO_TOOLS_WAIT_MS);
- timer.unref();
- });
+ let settled = false;
+ const delay = (ms: number) =>
+ new Promise<void>(resolve => {
+ if (settled) return resolve();
+ timer = setTimeout(resolve, ms);
+ timer.unref();
+ });
+ // Fused tools are worth waiting for past the raw lookup; appended ones are not.
+ const deadline = fuseTools
+ ? Promise.resolve(alongside).then(() =>
+ delay(config.ALEXANDRIA_TOOL_FUSION_WAIT_MS),
+ )
+ : Promise.all([delay(AUTO_TOOLS_WAIT_MS), alongside]);
try {
- return await Promise.race([
- extraDiscovery,
- Promise.all([deadline, alongside]).then(() => null),
- ]);
+ return await Promise.race([extraDiscovery, deadline.then(() => null)]);
} finally {
+ settled = true;
clearTimeout(timer);
}
};
@@ -457,35 +541,26 @@ export async function executeSearch(
baseDiscovery,
waitForExtra(baseDiscovery),
]);
- const merged = new Map(
- discovery.items.map(tool => [
- `${tool.provider}/${tool.capability}`,
- tool,
- ]),
- );
- for (const tool of extra?.items ?? []) {
- const key = `${tool.provider}/${tool.capability}`;
- const previous = merged.get(key);
- merged.set(
- key,
- previous && "matchedBy" in previous && "matchedBy" in tool
- ? {
- ...previous,
- matchedBy: [
- ...new Set([
- ...(previous.matchedBy ?? []),
- ...(tool.matchedBy ?? []),
- ]),
- ],
- matchedUrls: [
- ...new Set([
- ...(previous.matchedUrls ?? []),
- ...(tool.matchedUrls ?? []),
- ]),
- ],
- }
- : (previous ?? tool),
- );
+ const merged = new Map<string, DiscoveredTool>();
+ if (fuseTools && extra) {
+ // Semantic matches are fused; domain-only matches follow, as they do unfused.
+ const fits = new Set(extra.items.map(toolKey));
+ const baseSemantic = discovery.items.slice(0, discovery.semanticMatches);
+ const baseKeys = new Set(baseSemantic.map(toolKey));
+ for (const tool of [
+ ...fuseRankings(
+ [baseSemantic, extra.found.slice(0, extra.semanticMatches)],
+ tool => baseKeys.has(toolKey(tool)) || fits.has(toolKey(tool)),
+ ),
+ ...discovery.items.slice(baseSemantic.length),
+ ...extra.found
+ .slice(extra.semanticMatches)
+ .filter(tool => fits.has(toolKey(tool))),
+ ])
+ addTool(merged, tool);
+ } else {
+ for (const tool of [...discovery.items, ...(extra?.items ?? [])])
+ addTool(merged, tool);
}
searchResponse.tools = [...merged.values()];
// Both legs can hide the same tools, so the larger count is reported.
diff --git a/apps/api/src/search/tool-intent.test.ts b/apps/api/src/search/tool-intent.test.ts
index 1a35fbdaf..6993b9e92 100644
--- a/apps/api/src/search/tool-intent.test.ts
+++ b/apps/api/src/search/tool-intent.test.ts
@@ -10,6 +10,7 @@ const config = vi.hoisted(() => ({
ANTHROPIC_API_KEY: undefined as string | undefined,
TYPESAFE_API_KEY: "key" as string | undefined,
ALEXANDRIA_AUTO_TOOLS: false,
+ ALEXANDRIA_TOOL_FUSION: false,
}));
vi.mock("ai", () => ({ generateObject: mocks.generateObject }));
@@ -28,6 +29,7 @@ import {
inferToolIntent,
parseSearchQuery,
queryInferenceEnabled,
+ toolFusionEnabled,
verifyTools,
} from "./tool-intent";
@@ -124,6 +126,15 @@ describe("feature gates", () => {
expect(queryInferenceEnabled()).toBe(false);
config.TYPESAFE_API_KEY = "key";
});
+
+ it("keep tool fusion off unless the config or team flag turns it on", () => {
+ expect(toolFusionEnabled({})).toBe(false);
+ expect(toolFusionEnabled(null)).toBe(false);
+ expect(toolFusionEnabled({ alexandriaToolFusion: true })).toBe(true);
+ config.ALEXANDRIA_TOOL_FUSION = true;
+ expect(toolFusionEnabled({})).toBe(true);
+ config.ALEXANDRIA_TOOL_FUSION = false;
+ });
});
it("sends no semantic query for an operator-only query", async () => {
@@ -158,3 +169,16 @@ it("keeps only the tools Jev judges to fit, and drops unjudged ones", async () =
tools[0],
]);
});
+
+it("keeps tools at or above a lower fit threshold when one is given", async () => {
+ const tools = [
+ { provider: "fiscal-ai", capability: "ownership/holders", description: "" },
+ { provider: "sec-gov", capability: "filings/filings", description: "" },
+ ];
+ systemOne
+ .mockResolvedValueOnce({ answers: { fits: { noul: 0.2 } } })
+ .mockResolvedValueOnce({ answers: { fits: { noul: 0.1 } } });
+ expect(await verifyTools("who owns apple stock", tools, logger, 0.2)).toEqual(
+ [tools[0]],
+ );
+});
diff --git a/apps/api/src/search/tool-intent.ts b/apps/api/src/search/tool-intent.ts
index 5ced17a0f..beb926b5a 100644
--- a/apps/api/src/search/tool-intent.ts
+++ b/apps/api/src/search/tool-intent.ts
@@ -37,6 +37,9 @@ export const autoToolsEnabled = (flags: TeamFlags | null | undefined) =>
modelsConfigured() &&
(config.ALEXANDRIA_AUTO_TOOLS || flags?.alexandriaAutoTools === true);
+export const toolFusionEnabled = (flags: TeamFlags | null | undefined) =>
+ config.ALEXANDRIA_TOOL_FUSION || flags?.alexandriaToolFusion === true;
+
const OPERATORS = new Set([
"site",
"inurl",
@@ -264,6 +267,8 @@ export async function inferToolIntent(
const VERIFY_TIMEOUT_MS = 1500;
/** Jev's probability that a tool fits the query, at or above which it is kept. */
const FIT_THRESHOLD = 0.5;
+/** Lower bar for rewrite-found tools when they are fused into the raw-query ranking. */
+export const FUSED_FIT_THRESHOLD = 0.2;
const fitQuestion = {
fits: noul(
@@ -288,6 +293,7 @@ export async function verifyTools(
query: string,
tools: DiscoveredTool[],
logger: Logger,
+ threshold = FIT_THRESHOLD,
): Promise<DiscoveredTool[]> {
const typesafe = getTypeSafeClient();
if (!typesafe || !tools.length) return [];
@@ -309,7 +315,7 @@ export async function verifyTools(
},
{ signal, timeout: VERIFY_TIMEOUT_MS, retry: { maxRetries: 0 } },
);
- return answers.fits.noul >= FIT_THRESHOLD;
+ return answers.fits.noul >= threshold;
} catch {
failed++;
return false;