Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): expose Alexandria domain providers as scrape agent hint signals (#5091)

Rakshith Ramprakashc22afa500aea
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index b871c7e52..ddbf9502a 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -52,9 +52,15 @@ import { resolveThreatProtection } from "../../lib/threat-protection/request";
import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";
import { isAgentInteropSecretValid } from "../../lib/agent-interop";
import { markAlexandriaActivity } from "../../lib/alexandria-activity";
+import {
+ domainProvidersWithin,
+ lookupDomainProviders,
+} from "../../lib/alexandria-domain-providers";
+import { agentHintsRequested } from "../../middlewares/agent-hints";
import { DEFAULT_TEAM_LIMITS } from "../../services/autumn/autumn.service";
const AGENT_INTEROP_CONCURRENCY_BOOST = 3;
+const DOMAIN_PROVIDER_HINT_GRACE_MS = 100;
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
@@ -219,6 +225,13 @@ export async function scrapeController(
success: false,
error: "Provider discovery does not support zero data retention.",
});
+ const domainProviders =
+ agentHintsRequested(req) &&
+ !zeroDataRetention &&
+ !zeroDataRetentionTrace &&
+ !req.auth.team_id.startsWith("preview_keyless_")
+ ? lookupDomainProviders(req.body.url, req.auth.team_id)
+ : undefined;
const billing: BillingMetadata = {
...(req.body.__agentInterop
? { endpoint: "agent" as const, jobId }
@@ -776,6 +789,11 @@ export async function scrapeController(
})
: undefined;
if (tools) markAlexandriaActivity(req.auth.team_id);
+ if (domainProviders)
+ res.locals.agentHintDomainProviders = await domainProvidersWithin(
+ domainProviders,
+ DOMAIN_PROVIDER_HINT_GRACE_MS,
+ );
return res.status(200).json({
success: true,
diff --git a/apps/api/src/lib/agent-hint-signals.ts b/apps/api/src/lib/agent-hint-signals.ts
index afd154440..c4cdabb5a 100644
--- a/apps/api/src/lib/agent-hint-signals.ts
+++ b/apps/api/src/lib/agent-hint-signals.ts
@@ -14,6 +14,8 @@ export interface AgentHintSignalContext {
remainingCredits?: number;
canUseMapAndCrawl?: boolean;
canUseInteract?: boolean;
+ /** Alexandria providers covering the page's host, when that lookup succeeded in time; undefined when it was skipped, late or failed. */
+ domainProviders?: string[];
}
export type AgentHintSignalValue = string | number | boolean | string[];
@@ -230,6 +232,12 @@ export function computeAgentHintSignals(
addPageSignals(signals, response, object(data.metadata));
+ if (context.domainProviders) {
+ signals.set("alexandria_provider_count", context.domainProviders.length);
+ signals.set("alexandria_providers", context.domainProviders);
+ setDefined(signals, "alexandria_provider", context.domainProviders[0]);
+ }
+
const web = Array.isArray(response.data)
? response.data
: Array.isArray(data.web)
diff --git a/apps/api/src/lib/alexandria-domain-providers.test.ts b/apps/api/src/lib/alexandria-domain-providers.test.ts
new file mode 100644
index 000000000..a0348fd42
--- /dev/null
+++ b/apps/api/src/lib/alexandria-domain-providers.test.ts
@@ -0,0 +1,184 @@
+import { config } from "../config";
+import { exchangeRequest } from "../services/alexandria/client";
+import {
+ alexandriaDomainLookupsTotal,
+ clearDomainProviderCache,
+ domainProvidersWithin,
+ lookupDomainProviders,
+} from "./alexandria-domain-providers";
+import { computeAgentHintSignals } from "./agent-hint-signals";
+
+vi.mock("../services/alexandria/client", () => ({
+ exchangeRequest: vi.fn(),
+}));
+
+const exchange = vi.mocked(exchangeRequest);
+
+function resolves(ids: string[], delayMs = 0) {
+ exchange.mockImplementation(
+ () =>
+ new Promise(resolve =>
+ setTimeout(
+ () =>
+ resolve({
+ status: 200,
+ body: {
+ skills: ids.map(id => ({ id, matchedDomains: ["example"] })),
+ },
+ }),
+ delayMs,
+ ),
+ ),
+ );
+}
+
+async function count(outcome: string): Promise<number> {
+ const metric = await alexandriaDomainLookupsTotal.get();
+ return metric.values.find(v => v.labels.outcome === outcome)?.value ?? 0;
+}
+
+describe("Alexandria domain providers for agent hints", () => {
+ const originalUrl = config.FIRE_EXCHANGE_URL;
+
+ beforeEach(() => {
+ config.FIRE_EXCHANGE_URL = "http://exchange.test";
+ clearDomainProviderCache();
+ exchange.mockReset();
+ });
+
+ afterAll(() => {
+ config.FIRE_EXCHANGE_URL = originalUrl;
+ });
+
+ it("resolves the host only and serves later requests from the cache", async () => {
+ resolves(["finance-yahoo-com"]);
+ const first = lookupDomainProviders(
+ "https://Finance.Yahoo.com/quote/AAPL/?p=AAPL",
+ "team-1",
+ );
+ expect(await domainProvidersWithin(first!, 100)).toEqual([
+ "finance-yahoo-com",
+ ]);
+ expect(exchange).toHaveBeenCalledTimes(1);
+ expect(exchange.mock.calls[0][0]).toMatchObject({
+ path: "/v1/skills/resolve",
+ body: { urls: ["https://finance.yahoo.com/"] },
+ });
+
+ const hits = await count("cache_hit");
+ const second = lookupDomainProviders(
+ "https://finance.yahoo.com/quote/MSFT/",
+ "team-2",
+ );
+ expect(await domainProvidersWithin(second!, 0)).toEqual([
+ "finance-yahoo-com",
+ ]);
+ expect(exchange).toHaveBeenCalledTimes(1);
+ expect(await count("cache_hit")).toBe(hits + 1);
+ });
+
+ it("skips a late lookup but caches its result for the next request", async () => {
+ resolves(["github-com"], 150);
+ const late = await count("late");
+ const lookup = lookupDomainProviders(
+ "https://github.com/firecrawl/firecrawl",
+ "team-1",
+ );
+ expect(await domainProvidersWithin(lookup!, 20)).toBeUndefined();
+ expect(await count("late")).toBe(late + 1);
+
+ const concurrent = lookupDomainProviders(
+ "https://github.com/other/repo",
+ "team-1",
+ );
+ expect(concurrent).toBe(lookup);
+
+ await lookup;
+ const next = lookupDomainProviders("https://github.com/a/b", "team-1");
+ expect(await domainProvidersWithin(next!, 0)).toEqual(["github-com"]);
+ expect(exchange).toHaveBeenCalledTimes(1);
+ });
+
+ it("reports a failed lookup as unknown, not as no providers, and backs off", async () => {
+ exchange.mockRejectedValue(new Error("connect ECONNREFUSED"));
+ const errors = await count("error");
+ const lookup = lookupDomainProviders("https://example.com/", "team-1");
+ expect(await domainProvidersWithin(lookup!, 100)).toBeUndefined();
+ expect(await count("error")).toBe(errors + 1);
+
+ exchange.mockResolvedValue({ status: 503, body: "unavailable" });
+ const other = lookupDomainProviders("https://example.org/", "team-1");
+ expect(await domainProvidersWithin(other!, 100)).toBeUndefined();
+
+ const late = await count("late");
+ const again = lookupDomainProviders("https://example.com/x", "team-1");
+ expect(await domainProvidersWithin(again!, 0)).toBeUndefined();
+ expect(await count("late")).toBe(late);
+ expect(exchange).toHaveBeenCalledTimes(2);
+ });
+
+ it("caches a host with no matching provider", async () => {
+ resolves([]);
+ const lookup = lookupDomainProviders(
+ "https://www.paulgraham.com/greatwork.html",
+ "team-1",
+ );
+ expect(await domainProvidersWithin(lookup!, 100)).toEqual([]);
+ const again = lookupDomainProviders(
+ "https://www.paulgraham.com/hwh.html",
+ "team-1",
+ );
+ expect(await domainProvidersWithin(again!, 0)).toEqual([]);
+ expect(exchange).toHaveBeenCalledTimes(1);
+ });
+
+ it("drops provider ids that are not plain identifiers", async () => {
+ resolves(["wikipedia", "bad id\nwith text", "wikipedia"]);
+ const lookup = lookupDomainProviders(
+ "https://en.wikipedia.org/wiki/X",
+ "team-1",
+ );
+ expect(await domainProvidersWithin(lookup!, 100)).toEqual(["wikipedia"]);
+ });
+
+ it("does nothing without an Exchange or a usable URL", () => {
+ expect(
+ lookupDomainProviders("https://user:[email protected]/", "team-1"),
+ ).toBeUndefined();
+ expect(lookupDomainProviders("ftp://example.com/", "team-1")).toBe(
+ undefined,
+ );
+ expect(lookupDomainProviders(undefined, "team-1")).toBeUndefined();
+ config.FIRE_EXCHANGE_URL = undefined;
+ expect(
+ lookupDomainProviders("https://example.com/", "team-1"),
+ ).toBeUndefined();
+ expect(exchange).not.toHaveBeenCalled();
+ });
+
+ it("exposes providers as hint signals only when the lookup finished", () => {
+ const response = { success: true, data: { metadata: {} } };
+ const matched = computeAgentHintSignals({
+ endpoint: "scrape",
+ response,
+ domainProviders: ["finance-yahoo-com", "sec-gov"],
+ });
+ expect(matched.get("alexandria_provider_count")).toBe(2);
+ expect(matched.get("alexandria_provider")).toBe("finance-yahoo-com");
+ expect(matched.get("alexandria_providers")).toEqual([
+ "finance-yahoo-com",
+ "sec-gov",
+ ]);
+
+ const unmatched = computeAgentHintSignals({
+ endpoint: "scrape",
+ response,
+ domainProviders: [],
+ });
+ expect(unmatched.get("alexandria_provider_count")).toBe(0);
+ expect(unmatched.has("alexandria_provider")).toBe(false);
+
+ const skipped = computeAgentHintSignals({ endpoint: "scrape", response });
+ expect(skipped.has("alexandria_provider_count")).toBe(false);
+ });
+});
diff --git a/apps/api/src/lib/alexandria-domain-providers.ts b/apps/api/src/lib/alexandria-domain-providers.ts
new file mode 100644
index 000000000..11d587006
--- /dev/null
+++ b/apps/api/src/lib/alexandria-domain-providers.ts
@@ -0,0 +1,146 @@
+import { Counter } from "prom-client";
+import { z } from "zod";
+import { config } from "../config";
+import { exchangeRequest } from "../services/alexandria/client";
+import { logger } from "./logger";
+
+/**
+ * Alexandria providers whose tools cover a page's host, used only for agent
+ * hints. A lookup starts when a request arrives and runs alongside it; callers
+ * wait a bounded time for the result and skip it when it is late, so a lookup
+ * never delays or fails the request it serves.
+ *
+ * Only the host is sent, and results are cached per host: the API always
+ * requests the extended catalogue, so a host resolves to the same providers
+ * for every team. Unmatched hosts are cached as well. Failures resolve to
+ * undefined, never to an empty list, so an outage is not read as "no
+ * provider"; they are cached briefly so an unavailable Exchange is not called
+ * on every request.
+ */
+const MATCH_TTL_MS = 10 * 60_000;
+const ERROR_TTL_MS = 30_000;
+const LOOKUP_TIMEOUT_MS = 5_000;
+const MAX_CACHED_HOSTS = 10_000;
+const PROVIDER_ID = /^[a-z0-9][a-z0-9._-]{0,99}$/i;
+
+const resolveResponseSchema = z.object({
+ skills: z.array(z.object({ id: z.string() })),
+});
+
+type Providers = string[] | undefined;
+
+const cache = new Map<string, { providers: Providers; expiresAt: number }>();
+const inFlight = new Map<string, Promise<Providers>>();
+
+export const alexandriaDomainLookupsTotal = new Counter({
+ name: "alexandria_domain_lookups_total",
+ help: "Alexandria domain provider lookups for agent hints, by outcome",
+ labelNames: ["outcome"] as const,
+});
+
+function hostOf(url: unknown): string | undefined {
+ if (typeof url !== "string") return undefined;
+ const parsed = URL.parse(url);
+ if (
+ !parsed ||
+ !["http:", "https:"].includes(parsed.protocol) ||
+ parsed.username ||
+ parsed.password
+ )
+ return undefined;
+ return parsed.hostname.toLowerCase().replace(/\.$/, "") || undefined;
+}
+
+function remember(host: string, providers: Providers, ttlMs: number): void {
+ cache.delete(host);
+ if (cache.size >= MAX_CACHED_HOSTS) {
+ const oldest = cache.keys().next().value;
+ if (oldest !== undefined) cache.delete(oldest);
+ }
+ cache.set(host, { providers, expiresAt: Date.now() + ttlMs });
+}
+
+/**
+ * Starts (or reuses) the provider lookup for a URL's host. Returns undefined
+ * when Alexandria is not configured or the URL has no usable host. The
+ * returned promise never rejects: it resolves to the matching providers (an
+ * empty list when none match) or to undefined when the lookup failed.
+ */
+export function lookupDomainProviders(
+ url: unknown,
+ teamId: string,
+): Promise<Providers> | undefined {
+ if (!config.FIRE_EXCHANGE_URL) return undefined;
+ const host = hostOf(url);
+ if (!host) return undefined;
+
+ const cached = cache.get(host);
+ if (cached && cached.expiresAt > Date.now()) {
+ alexandriaDomainLookupsTotal.inc({ outcome: "cache_hit" });
+ return Promise.resolve(cached.providers);
+ }
+ const pending = inFlight.get(host);
+ if (pending) return pending;
+
+ alexandriaDomainLookupsTotal.inc({ outcome: "cache_miss" });
+ const lookup = exchangeRequest({
+ teamId,
+ path: "/v1/skills/resolve",
+ body: { urls: [`https://${host}/`] },
+ timeoutMs: LOOKUP_TIMEOUT_MS,
+ })
+ .then(result => {
+ if (result.status !== 200)
+ throw new Error(`Domain resolve returned ${result.status}`);
+ const providers = [
+ ...new Set(
+ resolveResponseSchema
+ .parse(result.body)
+ .skills.map(skill => skill.id)
+ .filter(id => PROVIDER_ID.test(id)),
+ ),
+ ];
+ remember(host, providers, MATCH_TTL_MS);
+ return providers;
+ })
+ .catch(error => {
+ alexandriaDomainLookupsTotal.inc({ outcome: "error" });
+ logger.warn("Alexandria domain lookup failed", {
+ module: "alexandria-domain-providers",
+ error: error instanceof Error ? error.message : String(error),
+ });
+ remember(host, undefined, ERROR_TTL_MS);
+ return undefined;
+ })
+ .finally(() => inFlight.delete(host));
+ inFlight.set(host, lookup);
+ return lookup;
+}
+
+/**
+ * The lookup's providers if it succeeds within `ms`; undefined when it is
+ * late or failed.
+ */
+export async function domainProvidersWithin(
+ lookup: Promise<Providers>,
+ ms: number,
+): Promise<Providers> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const late = Symbol("late");
+ const timeout = new Promise<typeof late>(resolve => {
+ timer = setTimeout(() => resolve(late), ms);
+ });
+ try {
+ const providers = await Promise.race([lookup, timeout]);
+ if (providers !== late) return providers;
+ alexandriaDomainLookupsTotal.inc({ outcome: "late" });
+ return undefined;
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
+export function clearDomainProviderCache(): void {
+ cache.clear();
+ inFlight.clear();
+}
diff --git a/apps/api/src/middlewares/__tests__/agent-hints.test.ts b/apps/api/src/middlewares/__tests__/agent-hints.test.ts
index 4a6eea6ac..23299a8b9 100644
--- a/apps/api/src/middlewares/__tests__/agent-hints.test.ts
+++ b/apps/api/src/middlewares/__tests__/agent-hints.test.ts
@@ -90,6 +90,7 @@ function appFor(
remainingCredits,
teamId = "account-team",
rules = TEST_RULES,
+ domainProviders,
} = {} as any,
) {
const app = express();
@@ -97,6 +98,7 @@ function appFor(
app.post("/", agentHintsMiddleware(endpoint), (req, res) => {
(req as any).auth = { team_id: teamId };
res.locals.agentCreditsRemaining = remainingCredits;
+ res.locals.agentHintDomainProviders = domainProviders;
res.locals.agentHintsProvider = { settled: true, hints: [], rules };
res.status(status).json(body);
});
@@ -196,6 +198,48 @@ describe("agent hint response middleware", () => {
expect(response.body.agent_hints).toEqual(["Interact: scrape-id."]);
});
+ it("passes matched domain providers to the rules and changes only agent_hints", async () => {
+ const body = {
+ success: true,
+ data: { markdown: "page", metadata: { statusCode: 200 } },
+ };
+ const rules: AgentHintRule[] = [
+ {
+ id: "domain-provider",
+ when: [
+ SUCCESS,
+ { signal: "alexandria_provider_count", op: "gt", value: 0 },
+ ],
+ text: "Provider: {alexandria_provider} of {alexandria_provider_count}.",
+ },
+ ];
+ const matched = await request(
+ appFor({
+ endpoint: "scrape",
+ body,
+ rules,
+ domainProviders: ["finance-yahoo-com", "sec-gov"],
+ }),
+ )
+ .post("/")
+ .set("X-Firecrawl-Agent-Hints", "true")
+ .send({});
+ expect(matched.body).toEqual({
+ ...body,
+ agent_hints: ["Provider: finance-yahoo-com of 2."],
+ });
+
+ for (const domainProviders of [[], undefined]) {
+ const response = await request(
+ appFor({ endpoint: "scrape", body, rules, domainProviders }),
+ )
+ .post("/")
+ .set("X-Firecrawl-Agent-Hints", "true")
+ .send({});
+ expect(response.body).toEqual(body);
+ }
+ });
+
it("reports Interact as unavailable without database authentication", async () => {
config.USE_DB_AUTHENTICATION = false;
const response = await request(
diff --git a/apps/api/src/middlewares/agent-hints.ts b/apps/api/src/middlewares/agent-hints.ts
index 0d3d4d1dc..35b5d810c 100644
--- a/apps/api/src/middlewares/agent-hints.ts
+++ b/apps/api/src/middlewares/agent-hints.ts
@@ -16,7 +16,7 @@ import {
import { keylessSignupSurface } from "../lib/keyless-signup-link";
import { logger } from "../lib/logger";
-function agentHintsRequested(req: Request): boolean {
+export function agentHintsRequested(req: Request): boolean {
return req.get("X-Firecrawl-Agent-Hints")?.trim().toLowerCase() === "true";
}
@@ -35,6 +35,7 @@ function agentHintsFor(
remainingCredits: res.locals.agentCreditsRemaining,
canUseMapAndCrawl: !!teamId && !teamId.startsWith("preview_keyless_"),
canUseInteract: config.USE_DB_AUTHENTICATION === true,
+ domainProviders: res.locals.agentHintDomainProviders,
};
const rules = provider?.settled ? provider.rules : [];
const { hints, providerHintIds } = mergeAgentHints(