Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

perf(api): start Alexandria tool lookup with the web search (#5332)

Rakshith Ramprakash0fa7e2cdd317
diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.ts
index 433d3973a..327450f4f 100644
--- a/apps/api/src/search/alexandria.ts
+++ b/apps/api/src/search/alexandria.ts
@@ -45,7 +45,8 @@ export async function discoverTools(
teamId: string;
toolDetail?: "compact" | "summary" | "full";
query?: string;
- urls?: string[];
+ /** May still be pending; the semantic lookup starts without waiting for it. */
+ urls?: string[] | Promise<string[]>;
limit: number;
timeoutMs: number;
/** The team may use its own MCP connectors: their listed tools join its discovery. */
@@ -53,20 +54,20 @@ export async function discoverTools(
},
logger: Logger,
): Promise<ToolDiscovery> {
- const deadline = Date.now() + Math.min(10000, input.timeoutMs);
+ const budget = () => Date.now() + Math.min(10000, input.timeoutMs);
const items = new Map<string, DetailedDiscoveredTool>();
const selected = { semantic: new Set<string>(), domain: new Set<string>() };
let attempted = 0;
let answered = 0;
- const remaining = () => {
+ const remaining = (deadline: number) => {
if (Date.now() >= deadline) throw new Error("Tool discovery timed out");
return deadline - Date.now();
};
- const lookup = async (options: Record<string, unknown>) => {
+ const lookup = async (options: Record<string, unknown>, deadline: number) => {
const result = await exchangeRequest({
teamId: input.teamId,
path: "/v1/retrieve",
- timeoutMs: remaining(),
+ timeoutMs: remaining(deadline),
maximumCredits: 0,
connectors: input.connectors === true,
body: {
@@ -125,43 +126,57 @@ export async function discoverTools(
});
}
};
- const leg = async (message: string, run: () => Promise<void>) => {
+ const leg = async <T>(
+ message: string,
+ deadline: number,
+ run: () => Promise<T>,
+ ): Promise<T | undefined> => {
attempted++;
for (let attempt = 1; ; attempt++) {
try {
- await run();
+ const value = await run();
answered++;
- return;
+ return value;
} catch (error) {
logger.warn(message, { error, attempt });
- if (attempt >= 2 || Date.now() >= deadline) return;
+ if (attempt >= 2 || Date.now() >= deadline) return undefined;
}
}
};
- if (input.query)
- await leg("Semantic tool discovery unavailable", async () => {
- for (const tool of await lookup({ query: input.query }))
- merge(tool, "semantic");
+ const semanticDeadline = budget();
+ const semantic = input.query
+ ? leg("Semantic tool discovery unavailable", semanticDeadline, () =>
+ lookup({ query: input.query }, semanticDeadline),
+ )
+ : undefined;
+ const domain = (async () => {
+ const urls = [...new Set((await input.urls) ?? [])].filter(value => {
+ const url = URL.parse(value);
+ return (
+ url &&
+ ["http:", "https:"].includes(url.protocol) &&
+ !url.username &&
+ !url.password &&
+ value.length <= 8192
+ );
});
- const urls = [...new Set(input.urls ?? [])].filter(value => {
- const url = URL.parse(value);
- return (
- url &&
- ["http:", "https:"].includes(url.protocol) &&
- !url.username &&
- !url.password &&
- value.length <= 8192
- );
- });
- if (urls.length)
- await leg("Domain tool discovery unavailable", async () => {
+ if (!urls.length) return undefined;
+ // The domain budget starts once the URLs exist, not while the web search runs.
+ const deadline = budget();
+ // Kept across retries so one failing provider does not discard the others.
+ const found: {
+ tool: z.infer<typeof toolSchema> | z.infer<typeof toolSummarySchema>;
+ urls: string[];
+ }[] = [];
+ await leg("Domain tool discovery unavailable", deadline, async () => {
// Resolve domains once; Exchange owns matching and provider visibility.
for (let i = 0; i < urls.length; i += 100) {
+ const batch = urls.slice(i, i + 100);
const result = await exchangeRequest({
teamId: input.teamId,
path: "/v1/skills/resolve",
- body: { urls: urls.slice(i, i + 100) },
- timeoutMs: remaining(),
+ body: { urls: batch },
+ timeoutMs: remaining(deadline),
connectors: input.connectors === true,
});
if (result.status !== 200)
@@ -181,50 +196,64 @@ export async function discoverTools(
.parse(result.body).skills;
const hostOf = (value: string) =>
new URL(value).hostname.toLowerCase().replace(/\.$/, "");
- for (const match of matches) {
- const matchedUrls = urls.slice(i, i + 100).filter(value => {
- const host = hostOf(value);
- return (
- match.matchedDomains.includes(host) ||
- match.matchedDomains.includes(host.replace(/^www\./, ""))
- );
- });
- const capabilitiesFor = (url: string) => {
- const host = hostOf(url);
- return [
- ...new Set([
- ...(match.domainCapabilities?.[url] ?? []),
- ...(match.domainCapabilities?.[host] ?? []),
- ...(match.domainCapabilities?.[host.replace(/^www\./, "")] ??
- []),
- ]),
+ const settled = await Promise.allSettled(
+ matches.map(async match => {
+ const matchedUrls = batch.filter(value => {
+ const host = hostOf(value);
+ return (
+ match.matchedDomains.includes(host) ||
+ match.matchedDomains.includes(host.replace(/^www\./, ""))
+ );
+ });
+ const capabilitiesFor = (url: string) => {
+ const host = hostOf(url);
+ return [
+ ...new Set([
+ ...(match.domainCapabilities?.[url] ?? []),
+ ...(match.domainCapabilities?.[host] ?? []),
+ ...(match.domainCapabilities?.[host.replace(/^www\./, "")] ??
+ []),
+ ]),
+ ];
+ };
+ const capabilities = [
+ ...new Set(matchedUrls.flatMap(capabilitiesFor)),
];
- };
- const capabilities = [
- ...new Set(matchedUrls.flatMap(capabilitiesFor)),
- ];
- if (
- !matchedUrls.length ||
- (match.domainCapabilities && !capabilities.length)
- )
- continue;
- for (const tool of await lookup({
- providers: [match.id],
- ...(capabilities.length ? { capabilities } : {}),
- }))
- if (tool.provider === match.id)
- merge(
+ if (
+ !matchedUrls.length ||
+ (match.domainCapabilities && !capabilities.length)
+ )
+ return [];
+ const tools = await lookup(
+ {
+ providers: [match.id],
+ ...(capabilities.length ? { capabilities } : {}),
+ },
+ deadline,
+ );
+ return tools
+ .filter(tool => tool.provider === match.id)
+ .map(tool => ({
tool,
- "domain",
- match.domainCapabilities
+ urls: match.domainCapabilities
? matchedUrls.filter(url =>
capabilitiesFor(url).includes(tool.capability),
)
: matchedUrls,
- );
- }
+ }));
+ }),
+ );
+ for (const list of settled)
+ if (list.status === "fulfilled") found.push(...list.value);
+ const failed = settled.find(list => list.status === "rejected");
+ if (failed) throw failed.reason;
}
});
+ return found;
+ })();
+ const [semanticTools, domainTools] = await Promise.all([semantic, domain]);
+ for (const tool of semanticTools ?? []) merge(tool, "semantic");
+ for (const { tool, urls } of domainTools ?? []) merge(tool, "domain", urls);
return {
items: [...items.values()].map(tool =>
input.toolDetail === "compact"
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index a97d74e76..d0734bb93 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -116,6 +116,29 @@ export async function executeSearch(
logger.info("Searching for results");
const wantsTools = sources.some(isAlexandriaSource);
+ let resolveToolUrls: (urls: string[]) => void = () => {};
+ const toolDiscovery =
+ !zeroDataRetention &&
+ !options.enterprise?.some(mode => mode === "zdr" || mode === "anon") &&
+ (wantsTools || options.domainTools)
+ ? discoverTools(
+ {
+ toolDetail: options.toolDetail ?? "compact",
+ teamId,
+ connectors: flags?.exchangeRetrieve === true,
+ limit,
+ query: wantsTools ? query : undefined,
+ urls:
+ options.domainTools === false
+ ? []
+ : new Promise<string[]>(resolve => {
+ resolveToolUrls = resolve;
+ }),
+ timeoutMs: options.timeout,
+ },
+ logger,
+ )
+ : null;
const searchTypes = [
...new Set(
sources
@@ -283,30 +306,15 @@ export async function executeSearch(
let toolsWarning: string | undefined;
let toolsUnavailable = false;
- if (
- !zeroDataRetention &&
- !options.enterprise?.some(mode => mode === "zdr" || mode === "anon") &&
- (wantsTools || options.domainTools)
- ) {
- const discovery = await discoverTools(
- {
- toolDetail: options.toolDetail ?? "compact",
- teamId,
- connectors: flags?.exchangeRetrieve === true,
- limit,
- query: wantsTools ? query : undefined,
- urls:
- options.domainTools === false
- ? []
- : [
- ...(searchResponse.web ?? []),
- ...(searchResponse.news ?? []),
- ...indexResults,
- ].flatMap(item => (item.url ? [item.url] : [])),
- timeoutMs: options.timeout,
- },
- logger,
+ if (toolDiscovery) {
+ resolveToolUrls(
+ [
+ ...(searchResponse.web ?? []),
+ ...(searchResponse.news ?? []),
+ ...indexResults,
+ ].flatMap(item => (item.url ? [item.url] : [])),
);
+ const discovery = await toolDiscovery;
searchResponse.tools = discovery.items;
toolsWarning = discovery.warning;
toolsUnavailable = discovery.unavailable === true;