Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
perf(api): start Alexandria tool lookup with the web search (#5332)
Rakshith Ramprakash0fa7e2cdd317
diff --git a/apps/api/src/search/alexandria.ts b/apps/api/src/search/alexandria.tsindex 433d3973a..327450f4f 100644--- a/apps/api/src/search/alexandria.ts+++ b/apps/api/src/search/alexandria.ts@@ -45,7 +45,8 @@ export async function discoverTools(teamId: string;toolDetail?: "compact" | "summary" | "full";query?: string;- urls?: string[];+ /** May still be pending; the semantic lookup starts without waiting for it. */+ urls?: string[] | Promise<string[]>;limit: number;timeoutMs: number;/** The team may use its own MCP connectors: their listed tools join its discovery. */@@ -53,20 +54,20 @@ export async function discoverTools(},logger: Logger,): Promise<ToolDiscovery> {- const deadline = Date.now() + Math.min(10000, input.timeoutMs);+ const budget = () => Date.now() + Math.min(10000, input.timeoutMs);const items = new Map<string, DetailedDiscoveredTool>();const selected = { semantic: new Set<string>(), domain: new Set<string>() };let attempted = 0;let answered = 0;- const remaining = () => {+ const remaining = (deadline: number) => {if (Date.now() >= deadline) throw new Error("Tool discovery timed out");return deadline - Date.now();};- const lookup = async (options: Record<string, unknown>) => {+ const lookup = async (options: Record<string, unknown>, deadline: number) => {const result = await exchangeRequest({teamId: input.teamId,path: "/v1/retrieve",- timeoutMs: remaining(),+ timeoutMs: remaining(deadline),maximumCredits: 0,connectors: input.connectors === true,body: {@@ -125,43 +126,57 @@ export async function discoverTools(});}};- const leg = async (message: string, run: () => Promise<void>) => {+ const leg = async <T>(+ message: string,+ deadline: number,+ run: () => Promise<T>,+ ): Promise<T | undefined> => {attempted++;for (let attempt = 1; ; attempt++) {try {- await run();+ const value = await run();answered++;- return;+ return value;} catch (error) {logger.warn(message, { error, attempt });- if (attempt >= 2 || Date.now() >= deadline) return;+ if (attempt >= 2 || Date.now() >= deadline) return undefined;}}};- if (input.query)- await leg("Semantic tool discovery unavailable", async () => {- for (const tool of await lookup({ query: input.query }))- merge(tool, "semantic");+ const semanticDeadline = budget();+ const semantic = input.query+ ? leg("Semantic tool discovery unavailable", semanticDeadline, () =>+ lookup({ query: input.query }, semanticDeadline),+ )+ : undefined;+ const domain = (async () => {+ const urls = [...new Set((await input.urls) ?? [])].filter(value => {+ const url = URL.parse(value);+ return (+ url &&+ ["http:", "https:"].includes(url.protocol) &&+ !url.username &&+ !url.password &&+ value.length <= 8192+ );});- const urls = [...new Set(input.urls ?? [])].filter(value => {- const url = URL.parse(value);- return (- url &&- ["http:", "https:"].includes(url.protocol) &&- !url.username &&- !url.password &&- value.length <= 8192- );- });- if (urls.length)- await leg("Domain tool discovery unavailable", async () => {+ if (!urls.length) return undefined;+ // The domain budget starts once the URLs exist, not while the web search runs.+ const deadline = budget();+ // Kept across retries so one failing provider does not discard the others.+ const found: {+ tool: z.infer<typeof toolSchema> | z.infer<typeof toolSummarySchema>;+ urls: string[];+ }[] = [];+ await leg("Domain tool discovery unavailable", deadline, async () => {// Resolve domains once; Exchange owns matching and provider visibility.for (let i = 0; i < urls.length; i += 100) {+ const batch = urls.slice(i, i + 100);const result = await exchangeRequest({teamId: input.teamId,path: "/v1/skills/resolve",- body: { urls: urls.slice(i, i + 100) },- timeoutMs: remaining(),+ body: { urls: batch },+ timeoutMs: remaining(deadline),connectors: input.connectors === true,});if (result.status !== 200)@@ -181,50 +196,64 @@ export async function discoverTools(.parse(result.body).skills;const hostOf = (value: string) =>new URL(value).hostname.toLowerCase().replace(/\.$/, "");- for (const match of matches) {- const matchedUrls = urls.slice(i, i + 100).filter(value => {- const host = hostOf(value);- return (- match.matchedDomains.includes(host) ||- match.matchedDomains.includes(host.replace(/^www\./, ""))- );- });- const capabilitiesFor = (url: string) => {- const host = hostOf(url);- return [- ...new Set([- ...(match.domainCapabilities?.[url] ?? []),- ...(match.domainCapabilities?.[host] ?? []),- ...(match.domainCapabilities?.[host.replace(/^www\./, "")] ??- []),- ]),+ const settled = await Promise.allSettled(+ matches.map(async match => {+ const matchedUrls = batch.filter(value => {+ const host = hostOf(value);+ return (+ match.matchedDomains.includes(host) ||+ match.matchedDomains.includes(host.replace(/^www\./, ""))+ );+ });+ const capabilitiesFor = (url: string) => {+ const host = hostOf(url);+ return [+ ...new Set([+ ...(match.domainCapabilities?.[url] ?? []),+ ...(match.domainCapabilities?.[host] ?? []),+ ...(match.domainCapabilities?.[host.replace(/^www\./, "")] ??+ []),+ ]),+ ];+ };+ const capabilities = [+ ...new Set(matchedUrls.flatMap(capabilitiesFor)),];- };- const capabilities = [- ...new Set(matchedUrls.flatMap(capabilitiesFor)),- ];- if (- !matchedUrls.length ||- (match.domainCapabilities && !capabilities.length)- )- continue;- for (const tool of await lookup({- providers: [match.id],- ...(capabilities.length ? { capabilities } : {}),- }))- if (tool.provider === match.id)- merge(+ if (+ !matchedUrls.length ||+ (match.domainCapabilities && !capabilities.length)+ )+ return [];+ const tools = await lookup(+ {+ providers: [match.id],+ ...(capabilities.length ? { capabilities } : {}),+ },+ deadline,+ );+ return tools+ .filter(tool => tool.provider === match.id)+ .map(tool => ({tool,- "domain",- match.domainCapabilities+ urls: match.domainCapabilities? matchedUrls.filter(url =>capabilitiesFor(url).includes(tool.capability),): matchedUrls,- );- }+ }));+ }),+ );+ for (const list of settled)+ if (list.status === "fulfilled") found.push(...list.value);+ const failed = settled.find(list => list.status === "rejected");+ if (failed) throw failed.reason;}});+ return found;+ })();+ const [semanticTools, domainTools] = await Promise.all([semantic, domain]);+ for (const tool of semanticTools ?? []) merge(tool, "semantic");+ for (const { tool, urls } of domainTools ?? []) merge(tool, "domain", urls);return {items: [...items.values()].map(tool =>input.toolDetail === "compact"diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex a97d74e76..d0734bb93 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -116,6 +116,29 @@ export async function executeSearch(logger.info("Searching for results");const wantsTools = sources.some(isAlexandriaSource);+ let resolveToolUrls: (urls: string[]) => void = () => {};+ const toolDiscovery =+ !zeroDataRetention &&+ !options.enterprise?.some(mode => mode === "zdr" || mode === "anon") &&+ (wantsTools || options.domainTools)+ ? discoverTools(+ {+ toolDetail: options.toolDetail ?? "compact",+ teamId,+ connectors: flags?.exchangeRetrieve === true,+ limit,+ query: wantsTools ? query : undefined,+ urls:+ options.domainTools === false+ ? []+ : new Promise<string[]>(resolve => {+ resolveToolUrls = resolve;+ }),+ timeoutMs: options.timeout,+ },+ logger,+ )+ : null;const searchTypes = [...new Set(sources@@ -283,30 +306,15 @@ export async function executeSearch(let toolsWarning: string | undefined;let toolsUnavailable = false;- if (- !zeroDataRetention &&- !options.enterprise?.some(mode => mode === "zdr" || mode === "anon") &&- (wantsTools || options.domainTools)- ) {- const discovery = await discoverTools(- {- toolDetail: options.toolDetail ?? "compact",- teamId,- connectors: flags?.exchangeRetrieve === true,- limit,- query: wantsTools ? query : undefined,- urls:- options.domainTools === false- ? []- : [- ...(searchResponse.web ?? []),- ...(searchResponse.news ?? []),- ...indexResults,- ].flatMap(item => (item.url ? [item.url] : [])),- timeoutMs: options.timeout,- },- logger,+ if (toolDiscovery) {+ resolveToolUrls(+ [+ ...(searchResponse.web ?? []),+ ...(searchResponse.news ?? []),+ ...indexResults,+ ].flatMap(item => (item.url ? [item.url] : [])),);+ const discovery = await toolDiscovery;searchResponse.tools = discovery.items;toolsWarning = discovery.warning;toolsUnavailable = discovery.unavailable === true;