Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

11.9 KB
Raw Blame
import { z } from "zod";
import type { Logger } from "winston";
import { developerCategoryAliases } from "../lib/developer-category";
import { exchangeRequest } from "../services/alexandria/client";
import {
  toolSchema,
  toolSummarySchema,
  type DiscoveredTool,
  type DetailedDiscoveredTool,
} from "../services/alexandria/contracts";

export const isAlexandriaSource = (source: { type: string }) =>
  source.type === "alexandria";
type ToolDiscovery = {
  items: DiscoveredTool[];
  warning?: string;
  /** Tools the Exchange hid from a US-only team; set only when it hid some. */
  usOnlyExcluded?: number;
  /** Every discovery leg failed and nothing was found. */
  unavailable?: true;
  /** How many leading items the semantic lookup found; the rest matched by domain only. */
  semanticMatches: number;
};

type SourceLike = string | { type: string };
type CategoryLike = string | { type: string };

const typeOf = (value: SourceLike | CategoryLike) =>
  typeof value === "string" ? value : value?.type;

const CATALOGUE_SOURCES = new Set(["alexandria", "skills"]);

function excludesIndexCategory(categories: unknown): boolean {
  return (
    Array.isArray(categories) &&
    categories.some(category => {
      const type = typeOf(category);
      return (
        type === "gov" ||
        (type !== undefined && developerCategoryAliases.has(type))
      );
    })
  );
}

export function isToolsOnlySearch(
  sources: unknown,
  categories: unknown,
): boolean {
  if (!Array.isArray(sources) || sources.length === 0) return false;
  if (!sources.every(source => typeOf(source) === "alexandria")) return false;
  if (excludesIndexCategory(categories)) return false;
  return true;
}

/**
 * Search that only asks the Alexandria catalogue for tools and/or skill cards.
 * Both are free, so the credit check is skipped the same way as a tools-only
 * search. An index category still has its own billable path.
 */
export function isFreeCatalogueSearch(
  sources: unknown,
  categories: unknown,
): boolean {
  if (!Array.isArray(sources) || sources.length === 0) return false;
  if (!sources.every(source => CATALOGUE_SOURCES.has(typeOf(source) ?? "")))
    return false;
  if (excludesIndexCategory(categories)) return false;
  return true;
}

export function usOnlyToolsWarning(excluded: number): string {
  return `${excluded} ${excluded === 1 ? "tool is" : "tools are"} not available for teams restricted to US-only processing.`;
}

export async function discoverTools(
  input: {
    teamId: string;
    toolDetail?: "compact" | "summary" | "full";
    /** Semantic queries, best first; their results are interleaved by rank. */
    queries?: string[];
    /** May still be pending; the semantic lookup starts without waiting for it. */
    urls?: string[] | Promise<string[]>;
    limit: number;
    timeoutMs: number;
    /** The team may use its own MCP connectors: their listed tools join its discovery. */
    connectors?: boolean;
    /** The team may only be served by US-based providers. */
    usOnly?: boolean;
    /** The team is held to zero data retention: only providers that support it are returned. */
    zeroDataRetention?: boolean;
    /** Names each find-tools lookup `<prefix>:<n>` and appends the id to `sent` as it goes out. */
    exchangeRequestIds?: { prefix: string; sent: string[] };
  },
  logger: Logger,
): Promise<ToolDiscovery> {
  const budget = () => Date.now() + Math.min(10000, input.timeoutMs);
  const items = new Map<string, DetailedDiscoveredTool>();
  const selected = { semantic: new Set<string>(), domain: new Set<string>() };
  let attempted = 0;
  let answered = 0;
  // Lookups can hide the same tools, so the largest count is reported.
  let usOnlyExcluded = 0;
  const remaining = (deadline: number) => {
    if (Date.now() >= deadline) throw new Error("Tool discovery timed out");
    return deadline - Date.now();
  };
  const lookup = async (options: Record<string, unknown>, deadline: number) => {
    const timeoutMs = remaining(deadline);
    const ids = input.exchangeRequestIds;
    const requestId = ids ? `${ids.prefix}:${ids.sent.length + 1}` : undefined;
    if (requestId) ids?.sent.push(requestId);
    const result = await exchangeRequest({
      teamId: input.teamId,
      path: "/v1/retrieve",
      requestId,
      timeoutMs,
      maximumCredits: 0,
      connectors: input.connectors === true,
      usOnly: input.usOnly === true,
      zeroDataRetention: input.zeroDataRetention === true,
      body: {
        provider: "firecrawl",
        capability: "find-tools",
        options: {
          ...options,
          level: "tools",
          expand:
            input.toolDetail === "full"
              ? ["options", "response", "examples"]
              : [],
          limit: Math.min(input.limit, 24),
        },
      },
    });
    if (result.status !== 200)
      throw new Error("Tool contract lookup unavailable");
    const data = z
      .object({
        success: z.literal(true),
        creditsCost: z.literal(0),
        data: z.object({
          items: z.array(
            input.toolDetail === "full" ? toolSchema : toolSummarySchema,
          ),
          usOnlyExcluded: z
            .number()
            .int()
            .nonnegative()
            .optional()
            .catch(undefined),
        }),
      })
      .parse(result.body).data;
    usOnlyExcluded = Math.max(usOnlyExcluded, data.usOnlyExcluded ?? 0);
    return data.items;
  };
  const merge = (
    tool: z.infer<typeof toolSchema> | z.infer<typeof toolSummarySchema>,
    source: "semantic" | "domain",
    urls: string[] = [],
  ) => {
    const id = `${tool.provider}/${tool.capability}`;
    const previous = items.get(id);
    if (
      !selected[source].has(id) &&
      selected[source].size >= Math.min(input.limit, 24)
    )
      return;
    selected[source].add(id);
    if (previous) {
      items.set(id, {
        ...previous,
        matchedBy: [...new Set([...(previous.matchedBy ?? []), source])],
        matchedUrls: [...new Set([...(previous.matchedUrls ?? []), ...urls])],
      });
    } else {
      items.set(id, {
        ...tool,
        id,
        matchedBy: [source],
        matchedUrls: [...new Set(urls)],
      });
    }
  };
  const leg = async <T>(
    message: string,
    deadline: number,
    run: () => Promise<T>,
  ): Promise<T | undefined> => {
    attempted++;
    for (let attempt = 1; ; attempt++) {
      try {
        const value = await run();
        answered++;
        return value;
      } catch (error) {
        logger.warn(message, { error, attempt });
        if (attempt >= 2 || Date.now() >= deadline) return undefined;
      }
    }
  };
  const queries = [...new Set(input.queries ?? [])];
  const semanticDeadline = budget();
  const semantic = queries.length
    ? leg("Semantic tool discovery unavailable", semanticDeadline, async () => {
        const settled = await Promise.allSettled(
          queries.map(query => lookup({ query }, semanticDeadline)),
        );
        const lists = settled.flatMap(result =>
          result.status === "fulfilled" ? [result.value] : [],
        );
        if (!lists.length) throw (settled[0] as PromiseRejectedResult).reason;
        const depth = Math.max(...lists.map(list => list.length));
        const ordered: (typeof lists)[number] = [];
        for (let rank = 0; rank < depth; rank++)
          for (const list of lists) if (list[rank]) ordered.push(list[rank]);
        return ordered;
      })
    : undefined;
  const domain = (async () => {
    const urls = [...new Set((await input.urls) ?? [])].filter(value => {
      const url = URL.parse(value);
      return (
        url &&
        ["http:", "https:"].includes(url.protocol) &&
        !url.username &&
        !url.password &&
        value.length <= 8192
      );
    });
    if (!urls.length) return undefined;
    // The domain budget starts once the URLs exist, not while the web search runs.
    const deadline = budget();
    // Kept across retries so one failing provider does not discard the others.
    const found: {
      tool: z.infer<typeof toolSchema> | z.infer<typeof toolSummarySchema>;
      urls: string[];
    }[] = [];
    await leg("Domain tool discovery unavailable", deadline, async () => {
      // Resolve domains once; Exchange owns matching and provider visibility.
      for (let i = 0; i < urls.length; i += 100) {
        const batch = urls.slice(i, i + 100);
        const result = await exchangeRequest({
          teamId: input.teamId,
          path: "/v1/skills/resolve",
          body: { urls: batch },
          timeoutMs: remaining(deadline),
          connectors: input.connectors === true,
          usOnly: input.usOnly === true,
          zeroDataRetention: input.zeroDataRetention === true,
        });
        if (result.status !== 200)
          throw new Error("Domain discovery unavailable");
        const matches = z
          .object({
            skills: z.array(
              z.object({
                id: z.string(),
                matchedDomains: z.array(z.string()),
                domainCapabilities: z
                  .record(z.string(), z.array(z.string()))
                  .optional(),
              }),
            ),
          })
          .parse(result.body).skills;
        const hostOf = (value: string) =>
          new URL(value).hostname.toLowerCase().replace(/\.$/, "");
        const settled = await Promise.allSettled(
          matches.map(async match => {
            const matchedUrls = batch.filter(value => {
              const host = hostOf(value);
              return (
                match.matchedDomains.includes(host) ||
                match.matchedDomains.includes(host.replace(/^www\./, ""))
              );
            });
            const capabilitiesFor = (url: string) => {
              const host = hostOf(url);
              return [
                ...new Set([
                  ...(match.domainCapabilities?.[url] ?? []),
                  ...(match.domainCapabilities?.[host] ?? []),
                  ...(match.domainCapabilities?.[host.replace(/^www\./, "")] ??
                    []),
                ]),
              ];
            };
            const capabilities = [
              ...new Set(matchedUrls.flatMap(capabilitiesFor)),
            ];
            if (
              !matchedUrls.length ||
              (match.domainCapabilities && !capabilities.length)
            )
              return [];
            const tools = await lookup(
              {
                providers: [match.id],
                ...(capabilities.length ? { capabilities } : {}),
              },
              deadline,
            );
            return tools
              .filter(tool => tool.provider === match.id)
              .map(tool => ({
                tool,
                urls: match.domainCapabilities
                  ? matchedUrls.filter(url =>
                      capabilitiesFor(url).includes(tool.capability),
                    )
                  : matchedUrls,
              }));
          }),
        );
        for (const list of settled)
          if (list.status === "fulfilled") found.push(...list.value);
        const failed = settled.find(list => list.status === "rejected");
        if (failed) throw failed.reason;
      }
    });
    return found;
  })();
  const [semanticTools, domainTools] = await Promise.all([semantic, domain]);
  for (const tool of semanticTools ?? []) merge(tool, "semantic");
  const semanticMatches = items.size;
  for (const { tool, urls } of domainTools ?? []) merge(tool, "domain", urls);
  return {
    items: [...items.values()].map(tool =>
      input.toolDetail === "compact"
        ? {
            provider: tool.provider,
            capability: tool.capability,
            description: tool.description,
          }
        : tool,
    ),
    semanticMatches,
    ...(answered < attempted
      ? { warning: "Some tool discovery results are unavailable." }
      : {}),
    ...(input.usOnly && usOnlyExcluded > 0 ? { usOnlyExcluded } : {}),
    ...(attempted && !answered && !items.size
      ? { unavailable: true as const }
      : {}),
  };
}