Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): expose Alexandria domain providers as scrape agent hint signals (#5091)
Rakshith Ramprakashc22afa500aea
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex b871c7e52..ddbf9502a 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -52,9 +52,15 @@ import { resolveThreatProtection } from "../../lib/threat-protection/request";import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";import { isAgentInteropSecretValid } from "../../lib/agent-interop";import { markAlexandriaActivity } from "../../lib/alexandria-activity";+import {+ domainProvidersWithin,+ lookupDomainProviders,+} from "../../lib/alexandria-domain-providers";+import { agentHintsRequested } from "../../middlewares/agent-hints";import { DEFAULT_TEAM_LIMITS } from "../../services/autumn/autumn.service";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;+const DOMAIN_PROVIDER_HINT_GRACE_MS = 100;export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -219,6 +225,13 @@ export async function scrapeController(success: false,error: "Provider discovery does not support zero data retention.",});+ const domainProviders =+ agentHintsRequested(req) &&+ !zeroDataRetention &&+ !zeroDataRetentionTrace &&+ !req.auth.team_id.startsWith("preview_keyless_")+ ? lookupDomainProviders(req.body.url, req.auth.team_id)+ : undefined;const billing: BillingMetadata = {...(req.body.__agentInterop? { endpoint: "agent" as const, jobId }@@ -776,6 +789,11 @@ export async function scrapeController(}): undefined;if (tools) markAlexandriaActivity(req.auth.team_id);+ if (domainProviders)+ res.locals.agentHintDomainProviders = await domainProvidersWithin(+ domainProviders,+ DOMAIN_PROVIDER_HINT_GRACE_MS,+ );return res.status(200).json({success: true,diff --git a/apps/api/src/lib/agent-hint-signals.ts b/apps/api/src/lib/agent-hint-signals.tsindex afd154440..c4cdabb5a 100644--- a/apps/api/src/lib/agent-hint-signals.ts+++ b/apps/api/src/lib/agent-hint-signals.ts@@ -14,6 +14,8 @@ export interface AgentHintSignalContext {remainingCredits?: number;canUseMapAndCrawl?: boolean;canUseInteract?: boolean;+ /** Alexandria providers covering the page's host, when that lookup succeeded in time; undefined when it was skipped, late or failed. */+ domainProviders?: string[];}export type AgentHintSignalValue = string | number | boolean | string[];@@ -230,6 +232,12 @@ export function computeAgentHintSignals(addPageSignals(signals, response, object(data.metadata));+ if (context.domainProviders) {+ signals.set("alexandria_provider_count", context.domainProviders.length);+ signals.set("alexandria_providers", context.domainProviders);+ setDefined(signals, "alexandria_provider", context.domainProviders[0]);+ }+const web = Array.isArray(response.data)? response.data: Array.isArray(data.web)diff --git a/apps/api/src/lib/alexandria-domain-providers.test.ts b/apps/api/src/lib/alexandria-domain-providers.test.tsnew file mode 100644index 000000000..a0348fd42--- /dev/null+++ b/apps/api/src/lib/alexandria-domain-providers.test.ts@@ -0,0 +1,184 @@+import { config } from "../config";+import { exchangeRequest } from "../services/alexandria/client";+import {+ alexandriaDomainLookupsTotal,+ clearDomainProviderCache,+ domainProvidersWithin,+ lookupDomainProviders,+} from "./alexandria-domain-providers";+import { computeAgentHintSignals } from "./agent-hint-signals";++vi.mock("../services/alexandria/client", () => ({+ exchangeRequest: vi.fn(),+}));++const exchange = vi.mocked(exchangeRequest);++function resolves(ids: string[], delayMs = 0) {+ exchange.mockImplementation(+ () =>+ new Promise(resolve =>+ setTimeout(+ () =>+ resolve({+ status: 200,+ body: {+ skills: ids.map(id => ({ id, matchedDomains: ["example"] })),+ },+ }),+ delayMs,+ ),+ ),+ );+}++async function count(outcome: string): Promise<number> {+ const metric = await alexandriaDomainLookupsTotal.get();+ return metric.values.find(v => v.labels.outcome === outcome)?.value ?? 0;+}++describe("Alexandria domain providers for agent hints", () => {+ const originalUrl = config.FIRE_EXCHANGE_URL;++ beforeEach(() => {+ config.FIRE_EXCHANGE_URL = "http://exchange.test";+ clearDomainProviderCache();+ exchange.mockReset();+ });++ afterAll(() => {+ config.FIRE_EXCHANGE_URL = originalUrl;+ });++ it("resolves the host only and serves later requests from the cache", async () => {+ resolves(["finance-yahoo-com"]);+ const first = lookupDomainProviders(+ "https://Finance.Yahoo.com/quote/AAPL/?p=AAPL",+ "team-1",+ );+ expect(await domainProvidersWithin(first!, 100)).toEqual([+ "finance-yahoo-com",+ ]);+ expect(exchange).toHaveBeenCalledTimes(1);+ expect(exchange.mock.calls[0][0]).toMatchObject({+ path: "/v1/skills/resolve",+ body: { urls: ["https://finance.yahoo.com/"] },+ });++ const hits = await count("cache_hit");+ const second = lookupDomainProviders(+ "https://finance.yahoo.com/quote/MSFT/",+ "team-2",+ );+ expect(await domainProvidersWithin(second!, 0)).toEqual([+ "finance-yahoo-com",+ ]);+ expect(exchange).toHaveBeenCalledTimes(1);+ expect(await count("cache_hit")).toBe(hits + 1);+ });++ it("skips a late lookup but caches its result for the next request", async () => {+ resolves(["github-com"], 150);+ const late = await count("late");+ const lookup = lookupDomainProviders(+ "https://github.com/firecrawl/firecrawl",+ "team-1",+ );+ expect(await domainProvidersWithin(lookup!, 20)).toBeUndefined();+ expect(await count("late")).toBe(late + 1);++ const concurrent = lookupDomainProviders(+ "https://github.com/other/repo",+ "team-1",+ );+ expect(concurrent).toBe(lookup);++ await lookup;+ const next = lookupDomainProviders("https://github.com/a/b", "team-1");+ expect(await domainProvidersWithin(next!, 0)).toEqual(["github-com"]);+ expect(exchange).toHaveBeenCalledTimes(1);+ });++ it("reports a failed lookup as unknown, not as no providers, and backs off", async () => {+ exchange.mockRejectedValue(new Error("connect ECONNREFUSED"));+ const errors = await count("error");+ const lookup = lookupDomainProviders("https://example.com/", "team-1");+ expect(await domainProvidersWithin(lookup!, 100)).toBeUndefined();+ expect(await count("error")).toBe(errors + 1);++ exchange.mockResolvedValue({ status: 503, body: "unavailable" });+ const other = lookupDomainProviders("https://example.org/", "team-1");+ expect(await domainProvidersWithin(other!, 100)).toBeUndefined();++ const late = await count("late");+ const again = lookupDomainProviders("https://example.com/x", "team-1");+ expect(await domainProvidersWithin(again!, 0)).toBeUndefined();+ expect(await count("late")).toBe(late);+ expect(exchange).toHaveBeenCalledTimes(2);+ });++ it("caches a host with no matching provider", async () => {+ resolves([]);+ const lookup = lookupDomainProviders(+ "https://www.paulgraham.com/greatwork.html",+ "team-1",+ );+ expect(await domainProvidersWithin(lookup!, 100)).toEqual([]);+ const again = lookupDomainProviders(+ "https://www.paulgraham.com/hwh.html",+ "team-1",+ );+ expect(await domainProvidersWithin(again!, 0)).toEqual([]);+ expect(exchange).toHaveBeenCalledTimes(1);+ });++ it("drops provider ids that are not plain identifiers", async () => {+ resolves(["wikipedia", "bad id\nwith text", "wikipedia"]);+ const lookup = lookupDomainProviders(+ "https://en.wikipedia.org/wiki/X",+ "team-1",+ );+ expect(await domainProvidersWithin(lookup!, 100)).toEqual(["wikipedia"]);+ });++ it("does nothing without an Exchange or a usable URL", () => {+ expect(+ lookupDomainProviders("https://user:[email protected]/", "team-1"),+ ).toBeUndefined();+ expect(lookupDomainProviders("ftp://example.com/", "team-1")).toBe(+ undefined,+ );+ expect(lookupDomainProviders(undefined, "team-1")).toBeUndefined();+ config.FIRE_EXCHANGE_URL = undefined;+ expect(+ lookupDomainProviders("https://example.com/", "team-1"),+ ).toBeUndefined();+ expect(exchange).not.toHaveBeenCalled();+ });++ it("exposes providers as hint signals only when the lookup finished", () => {+ const response = { success: true, data: { metadata: {} } };+ const matched = computeAgentHintSignals({+ endpoint: "scrape",+ response,+ domainProviders: ["finance-yahoo-com", "sec-gov"],+ });+ expect(matched.get("alexandria_provider_count")).toBe(2);+ expect(matched.get("alexandria_provider")).toBe("finance-yahoo-com");+ expect(matched.get("alexandria_providers")).toEqual([+ "finance-yahoo-com",+ "sec-gov",+ ]);++ const unmatched = computeAgentHintSignals({+ endpoint: "scrape",+ response,+ domainProviders: [],+ });+ expect(unmatched.get("alexandria_provider_count")).toBe(0);+ expect(unmatched.has("alexandria_provider")).toBe(false);++ const skipped = computeAgentHintSignals({ endpoint: "scrape", response });+ expect(skipped.has("alexandria_provider_count")).toBe(false);+ });+});diff --git a/apps/api/src/lib/alexandria-domain-providers.ts b/apps/api/src/lib/alexandria-domain-providers.tsnew file mode 100644index 000000000..11d587006--- /dev/null+++ b/apps/api/src/lib/alexandria-domain-providers.ts@@ -0,0 +1,146 @@+import { Counter } from "prom-client";+import { z } from "zod";+import { config } from "../config";+import { exchangeRequest } from "../services/alexandria/client";+import { logger } from "./logger";++/**+ * Alexandria providers whose tools cover a page's host, used only for agent+ * hints. A lookup starts when a request arrives and runs alongside it; callers+ * wait a bounded time for the result and skip it when it is late, so a lookup+ * never delays or fails the request it serves.+ *+ * Only the host is sent, and results are cached per host: the API always+ * requests the extended catalogue, so a host resolves to the same providers+ * for every team. Unmatched hosts are cached as well. Failures resolve to+ * undefined, never to an empty list, so an outage is not read as "no+ * provider"; they are cached briefly so an unavailable Exchange is not called+ * on every request.+ */+const MATCH_TTL_MS = 10 * 60_000;+const ERROR_TTL_MS = 30_000;+const LOOKUP_TIMEOUT_MS = 5_000;+const MAX_CACHED_HOSTS = 10_000;+const PROVIDER_ID = /^[a-z0-9][a-z0-9._-]{0,99}$/i;++const resolveResponseSchema = z.object({+ skills: z.array(z.object({ id: z.string() })),+});++type Providers = string[] | undefined;++const cache = new Map<string, { providers: Providers; expiresAt: number }>();+const inFlight = new Map<string, Promise<Providers>>();++export const alexandriaDomainLookupsTotal = new Counter({+ name: "alexandria_domain_lookups_total",+ help: "Alexandria domain provider lookups for agent hints, by outcome",+ labelNames: ["outcome"] as const,+});++function hostOf(url: unknown): string | undefined {+ if (typeof url !== "string") return undefined;+ const parsed = URL.parse(url);+ if (+ !parsed ||+ !["http:", "https:"].includes(parsed.protocol) ||+ parsed.username ||+ parsed.password+ )+ return undefined;+ return parsed.hostname.toLowerCase().replace(/\.$/, "") || undefined;+}++function remember(host: string, providers: Providers, ttlMs: number): void {+ cache.delete(host);+ if (cache.size >= MAX_CACHED_HOSTS) {+ const oldest = cache.keys().next().value;+ if (oldest !== undefined) cache.delete(oldest);+ }+ cache.set(host, { providers, expiresAt: Date.now() + ttlMs });+}++/**+ * Starts (or reuses) the provider lookup for a URL's host. Returns undefined+ * when Alexandria is not configured or the URL has no usable host. The+ * returned promise never rejects: it resolves to the matching providers (an+ * empty list when none match) or to undefined when the lookup failed.+ */+export function lookupDomainProviders(+ url: unknown,+ teamId: string,+): Promise<Providers> | undefined {+ if (!config.FIRE_EXCHANGE_URL) return undefined;+ const host = hostOf(url);+ if (!host) return undefined;++ const cached = cache.get(host);+ if (cached && cached.expiresAt > Date.now()) {+ alexandriaDomainLookupsTotal.inc({ outcome: "cache_hit" });+ return Promise.resolve(cached.providers);+ }+ const pending = inFlight.get(host);+ if (pending) return pending;++ alexandriaDomainLookupsTotal.inc({ outcome: "cache_miss" });+ const lookup = exchangeRequest({+ teamId,+ path: "/v1/skills/resolve",+ body: { urls: [`https://${host}/`] },+ timeoutMs: LOOKUP_TIMEOUT_MS,+ })+ .then(result => {+ if (result.status !== 200)+ throw new Error(`Domain resolve returned ${result.status}`);+ const providers = [+ ...new Set(+ resolveResponseSchema+ .parse(result.body)+ .skills.map(skill => skill.id)+ .filter(id => PROVIDER_ID.test(id)),+ ),+ ];+ remember(host, providers, MATCH_TTL_MS);+ return providers;+ })+ .catch(error => {+ alexandriaDomainLookupsTotal.inc({ outcome: "error" });+ logger.warn("Alexandria domain lookup failed", {+ module: "alexandria-domain-providers",+ error: error instanceof Error ? error.message : String(error),+ });+ remember(host, undefined, ERROR_TTL_MS);+ return undefined;+ })+ .finally(() => inFlight.delete(host));+ inFlight.set(host, lookup);+ return lookup;+}++/**+ * The lookup's providers if it succeeds within `ms`; undefined when it is+ * late or failed.+ */+export async function domainProvidersWithin(+ lookup: Promise<Providers>,+ ms: number,+): Promise<Providers> {+ let timer: ReturnType<typeof setTimeout> | undefined;+ const late = Symbol("late");+ const timeout = new Promise<typeof late>(resolve => {+ timer = setTimeout(() => resolve(late), ms);+ });+ try {+ const providers = await Promise.race([lookup, timeout]);+ if (providers !== late) return providers;+ alexandriaDomainLookupsTotal.inc({ outcome: "late" });+ return undefined;+ } finally {+ clearTimeout(timer);+ }+}++export function clearDomainProviderCache(): void {+ cache.clear();+ inFlight.clear();+}diff --git a/apps/api/src/middlewares/__tests__/agent-hints.test.ts b/apps/api/src/middlewares/__tests__/agent-hints.test.tsindex 4a6eea6ac..23299a8b9 100644--- a/apps/api/src/middlewares/__tests__/agent-hints.test.ts+++ b/apps/api/src/middlewares/__tests__/agent-hints.test.ts@@ -90,6 +90,7 @@ function appFor(remainingCredits,teamId = "account-team",rules = TEST_RULES,+ domainProviders,} = {} as any,) {const app = express();@@ -97,6 +98,7 @@ function appFor(app.post("/", agentHintsMiddleware(endpoint), (req, res) => {(req as any).auth = { team_id: teamId };res.locals.agentCreditsRemaining = remainingCredits;+ res.locals.agentHintDomainProviders = domainProviders;res.locals.agentHintsProvider = { settled: true, hints: [], rules };res.status(status).json(body);});@@ -196,6 +198,48 @@ describe("agent hint response middleware", () => {expect(response.body.agent_hints).toEqual(["Interact: scrape-id."]);});+ it("passes matched domain providers to the rules and changes only agent_hints", async () => {+ const body = {+ success: true,+ data: { markdown: "page", metadata: { statusCode: 200 } },+ };+ const rules: AgentHintRule[] = [+ {+ id: "domain-provider",+ when: [+ SUCCESS,+ { signal: "alexandria_provider_count", op: "gt", value: 0 },+ ],+ text: "Provider: {alexandria_provider} of {alexandria_provider_count}.",+ },+ ];+ const matched = await request(+ appFor({+ endpoint: "scrape",+ body,+ rules,+ domainProviders: ["finance-yahoo-com", "sec-gov"],+ }),+ )+ .post("/")+ .set("X-Firecrawl-Agent-Hints", "true")+ .send({});+ expect(matched.body).toEqual({+ ...body,+ agent_hints: ["Provider: finance-yahoo-com of 2."],+ });++ for (const domainProviders of [[], undefined]) {+ const response = await request(+ appFor({ endpoint: "scrape", body, rules, domainProviders }),+ )+ .post("/")+ .set("X-Firecrawl-Agent-Hints", "true")+ .send({});+ expect(response.body).toEqual(body);+ }+ });+it("reports Interact as unavailable without database authentication", async () => {config.USE_DB_AUTHENTICATION = false;const response = await request(diff --git a/apps/api/src/middlewares/agent-hints.ts b/apps/api/src/middlewares/agent-hints.tsindex 0d3d4d1dc..35b5d810c 100644--- a/apps/api/src/middlewares/agent-hints.ts+++ b/apps/api/src/middlewares/agent-hints.ts@@ -16,7 +16,7 @@ import {import { keylessSignupSurface } from "../lib/keyless-signup-link";import { logger } from "../lib/logger";-function agentHintsRequested(req: Request): boolean {+export function agentHintsRequested(req: Request): boolean {return req.get("X-Firecrawl-Agent-Hints")?.trim().toLowerCase() === "true";}@@ -35,6 +35,7 @@ function agentHintsFor(remainingCredits: res.locals.agentCreditsRemaining,canUseMapAndCrawl: !!teamId && !teamId.startsWith("preview_keyless_"),canUseInteract: config.USE_DB_AUTHENTICATION === true,+ domainProviders: res.locals.agentHintDomainProviders,};const rules = provider?.settled ? provider.rules : [];const { hints, providerHintIds } = mergeAgentHints(