Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Remove optional external guidance provider integration (#5104)
Rakshith Ramprakashf4fdf7f30041
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 594cf7518..ee3d8a3fb 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -278,17 +278,6 @@ jobs:--reporter=defaultworking-directory: apps/api- - name: Run agent hints regression tests- if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'- run: >-- pnpm exec vitest run- src/lib/agent-hints-provider.test.ts- src/lib/agent-hint-rules.test.ts- src/lib/agent-hint-signals.test.ts- src/middlewares/__tests__/agent-hints.test.ts- --reporter=default- working-directory: apps/api-- name: Run agent interop auth and concurrency regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: >-diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 76114a948..d9265d931 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -181,12 +181,3 @@ X402_ENDPOINT_PRICE_USD=0.01# WebP, AVIF) as one-page documents through FirePDF. Requires FIRE_PDF_BASE_URL.# Off by default; when off, image URLs fail as unsupported files.# IMAGE_OCR_ENABLED=true--# === External agent hints provider (optional) ===-# POST endpoint that supplies agent_hints guidance for hint-enabled requests.-# Disabled when unset.-# AGENT_HINTS_PROVIDER_URL=-# AGENT_HINTS_PROVIDER_SECRET=-# AGENT_HINTS_PROVIDER_TIMEOUT_MS=50-# Keys the pseudonymous team_id sent for keyless callers; never sent to the provider.-# AGENT_HINTS_PROVIDER_PSEUDONYM_KEY=diff --git a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts b/apps/api/src/__tests__/routes/check-credits-middleware.test.tsindex 724fc8746..585b1976c 100644--- a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts+++ b/apps/api/src/__tests__/routes/check-credits-middleware.test.ts@@ -107,8 +107,6 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {expect(res.status).not.toHaveBeenCalled();expect(req.account.remainingCredits).toBe(Infinity);- expect(res.locals.agentCreditsRemaining).toBe(0);- expect(res.locals.insufficientCredits).toBeUndefined();// request body limit must NOT have been clamped down to 0expect(req.body.limit).toBe(100);});@@ -120,8 +118,6 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {const { res } = await runMiddleware(req);expect(res.status).toHaveBeenCalledWith(402);- expect(res.locals.agentCreditsRemaining).toBe(0);- expect(res.locals.insufficientCredits).toBe(true);expect(logger.error).toHaveBeenCalledWith(expect.any(String),expect.not.objectContaining({ request: expect.anything() }),@@ -137,7 +133,6 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {const { res } = await runMiddleware(req);expect(res.status).not.toHaveBeenCalled();- expect(res.locals.agentCreditsRemaining).toBe(5);expect(req.body.limit).toBe(5);expect(logger.warn).toHaveBeenCalledWith("Adjusting limit to remaining credits",diff --git a/apps/api/src/__tests__/snips/v2/agent-hints.test.ts b/apps/api/src/__tests__/snips/v2/agent-hints.test.tsdeleted file mode 100644index 614aa8418..000000000--- a/apps/api/src/__tests__/snips/v2/agent-hints.test.ts+++ /dev/null@@ -1,91 +0,0 @@-import { describeIf, TEST_PRODUCTION, TEST_SUITE_WEBSITE } from "../lib";-import request, { idmux, Identity, scrapeTimeout, TEST_API_URL } from "./lib";--let identity: Identity;-beforeAll(async () => {- identity = await idmux({- name: "agent-hints",- concurrency: 10,- credits: 1000,- });-}, 20000);--describeIf(TEST_PRODUCTION)("Agent hints", () => {- const missingPage = `${TEST_SUITE_WEBSITE}/agent-hints-not-found`;-- it(- "keeps a missing source page response intact with hints enabled",- async () => {- const response = await request(TEST_API_URL)- .post("/v2/scrape")- .set("Authorization", `Bearer ${identity.apiKey}`)- .set("X-Firecrawl-Agent-Hints", "true")- .send({ url: missingPage, timeout: scrapeTimeout });- expect(response.statusCode).toBe(200);- expect(response.body.success).toBe(true);- expect(response.body.data.metadata.statusCode).toBe(404);- if (response.body.agent_hints !== undefined) {- expect(response.body.agent_hints).toEqual(- expect.arrayContaining([expect.any(String)]),- );- }- },- scrapeTimeout + 10000,- );-- it(- "keeps hints off when the opt-in header is false despite a hint condition",- async () => {- const response = await request(TEST_API_URL)- .post("/v2/scrape")- .set("Authorization", `Bearer ${identity.apiKey}`)- .set("X-Firecrawl-Agent-Hints", "false")- .send({ url: missingPage, timeout: scrapeTimeout });- expect(response.statusCode).toBe(200);- expect(response.body.success).toBe(true);- expect(response.body.data.metadata.statusCode).toBe(404);- expect(response.body).not.toHaveProperty("agent_hints");- },- scrapeTimeout + 10000,- );-- it(- "does not add static feedback guidance to a completed scrape",- async () => {- const response = await request(TEST_API_URL)- .post("/v2/scrape")- .set("Authorization", `Bearer ${identity.apiKey}`)- .set("X-Firecrawl-Agent-Hints", "true")- .send({ url: TEST_SUITE_WEBSITE, timeout: scrapeTimeout });- expect(response.statusCode).toBe(200);- expect(response.body.success).toBe(true);- expect(response.body).not.toHaveProperty("agent_hints");- },- scrapeTimeout + 10000,- );-- it(- "leaves hints off by default on a completed map",- async () => {- const response = await request(TEST_API_URL)- .post("/v2/map")- .set("Authorization", `Bearer ${identity.apiKey}`)- .send({ url: TEST_SUITE_WEBSITE, limit: 1, timeout: scrapeTimeout });- expect(response.statusCode).toBe(200);- expect(response.body).not.toHaveProperty("agent_hints");- },- scrapeTimeout + 10000,- );-- it("preserves validation errors without inventing hints", async () => {- const response = await request(TEST_API_URL)- .post("/v2/scrape")- .set("Authorization", `Bearer ${identity.apiKey}`)- .set("X-Firecrawl-Agent-Hints", "true")- .send({ url: "not-a-url" });- expect(response.statusCode).toBe(400);- expect(response.body.success).toBe(false);- expect(typeof response.body.error).toBe("string");- expect(response.body).not.toHaveProperty("agent_hints");- });-});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex df97dbafc..a25b5fb45 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -90,26 +90,6 @@ const configSchema = z.object({v => (typeof v === "string" && v.trim() === "" ? undefined : v),z.string().trim().optional(),),- // Optional external agent hints provider. Inert unless the URL is set.- AGENT_HINTS_PROVIDER_URL: emptyStringAsUndefined(- z- .string()- .url()- .refine(value => ["http:", "https:"].includes(new URL(value).protocol), {- message: "AGENT_HINTS_PROVIDER_URL must be an http(s) URL",- }),- ),- AGENT_HINTS_PROVIDER_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),- // Never sent to the provider. Keys the keyless team pseudonym; when unset a- // random per-process key is used, so pseudonyms are only stable per process.- AGENT_HINTS_PROVIDER_PSEUDONYM_KEY: emptyStringAsUndefined(- z.string().trim().min(32),- ),- AGENT_HINTS_PROVIDER_TIMEOUT_MS: z.coerce- .number()- .int()- .positive()- .default(50),RESEARCH_PROXY_URL: z.string().url().optional(),SEARCH_PLATFORM_URL: z.string().url().optional(),RESEARCH_KEYLESS_DISABLED: researchKeylessDisabled,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex ddbf9502a..b871c7e52 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -52,15 +52,9 @@ import { resolveThreatProtection } from "../../lib/threat-protection/request";import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";import { isAgentInteropSecretValid } from "../../lib/agent-interop";import { markAlexandriaActivity } from "../../lib/alexandria-activity";-import {- domainProvidersWithin,- lookupDomainProviders,-} from "../../lib/alexandria-domain-providers";-import { agentHintsRequested } from "../../middlewares/agent-hints";import { DEFAULT_TEAM_LIMITS } from "../../services/autumn/autumn.service";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;-const DOMAIN_PROVIDER_HINT_GRACE_MS = 100;export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -225,13 +219,6 @@ export async function scrapeController(success: false,error: "Provider discovery does not support zero data retention.",});- const domainProviders =- agentHintsRequested(req) &&- !zeroDataRetention &&- !zeroDataRetentionTrace &&- !req.auth.team_id.startsWith("preview_keyless_")- ? lookupDomainProviders(req.body.url, req.auth.team_id)- : undefined;const billing: BillingMetadata = {...(req.body.__agentInterop? { endpoint: "agent" as const, jobId }@@ -789,11 +776,6 @@ export async function scrapeController(}): undefined;if (tools) markAlexandriaActivity(req.auth.team_id);- if (domainProviders)- res.locals.agentHintDomainProviders = await domainProvidersWithin(- domainProviders,- DOMAIN_PROVIDER_HINT_GRACE_MS,- );return res.status(200).json({success: true,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 280c822c5..5638825d4 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1630,7 +1630,6 @@ export type VideoItem = {};export type ErrorResponse = {- agent_hints?: string[];success: false;code?: ErrorCodes;error: string;@@ -1643,7 +1642,6 @@ export type ScrapeResponse =| ErrorResponse| {success: true;- agent_hints?: string[];warning?: string;data: Document & {tools?: import("../../services/alexandria/contracts").DiscoveredTool[];@@ -2009,7 +2007,6 @@ export type MapResponse =| ErrorResponse| {success: true;- agent_hints?: string[];id: string;links?: MapDocument[];warning?: string;@@ -2770,7 +2767,6 @@ export type SearchResponse =| ErrorResponse| {success: true;- agent_hints?: string[];warning?: string;data: Document[];creditsUsed: number;@@ -2778,7 +2774,6 @@ export type SearchResponse =}| {success: true;- agent_hints?: string[];warning?: string;data: import("../../lib/entities").SearchV2Response;creditsUsed: number;@@ -2786,7 +2781,6 @@ export type SearchResponse =}| {success: true;- agent_hints?: string[];warning?: string;data: import("../../lib/entities").SearchV2Response;scrapeIds: {diff --git a/apps/api/src/lib/agent-hint-rules.test.ts b/apps/api/src/lib/agent-hint-rules.test.tsdeleted file mode 100644index a4a08e3d9..000000000--- a/apps/api/src/lib/agent-hint-rules.test.ts+++ /dev/null@@ -1,253 +0,0 @@-import {- evaluateAgentHintRules,- parseAgentHintRules,- type AgentHintRule,-} from "./agent-hint-rules";-import type { AgentHintSignalValue } from "./agent-hint-signals";--const signals = (values: Record<string, AgentHintSignalValue>) =>- new Map(Object.entries(values));--const rule = (- id: string,- when: AgentHintRule["when"],- text: string,- group?: string,-): AgentHintRule => (group ? { id, group, when, text } : { id, when, text });--describe("agent hint rule matcher", () => {- it.each([- ["eq", 404, 404, true],- ["eq", 404, "404", false],- ["ne", 404, 410, true],- ["ne", 404, 404, false],- ["lt", 99, 100, true],- ["lt", 100, 100, false],- ["lte", 100, 100, true],- ["gt", 4, 3, true],- ["gt", 3, 3, false],- ["gte", 0.75, 0.75, true],- ["gte", 0.74, 0.75, false],- ["lt", "1", 100, false],- ] as const)("%s: %p against %p holds = %p", (op, actual, value, expected) => {- const result = evaluateAgentHintRules(- [rule("r", [{ signal: "s", op, value } as any], "fired")],- signals({ s: actual }),- );- expect(result).toEqual(expected ? ["fired"] : []);- });-- it("matches in against a list of scalars", () => {- const rules = [- rule("r", [{ signal: "s", op: "in", value: [404, 410] }], "fired"),- ];- expect(evaluateAgentHintRules(rules, signals({ s: 410 }))).toEqual([- "fired",- ]);- expect(evaluateAgentHintRules(rules, signals({ s: 401 }))).toEqual([]);- });-- it("only lets exists hold for absent signals", () => {- for (const op of ["eq", "ne", "lt", "in"] as const) {- const value = op === "in" ? [1] : 1;- expect(- evaluateAgentHintRules(- [rule("r", [{ signal: "missing", op, value } as any], "fired")],- signals({}),- ),- ).toEqual([]);- }- const exists = (value: boolean) =>- rule("r", [{ signal: "s", op: "exists", value }], "fired");- expect(evaluateAgentHintRules([exists(false)], signals({}))).toEqual([- "fired",- ]);- expect(evaluateAgentHintRules([exists(true)], signals({}))).toEqual([]);- expect(evaluateAgentHintRules([exists(true)], signals({ s: 0 }))).toEqual([- "fired",- ]);- });-- it("never matches list signals with comparison operators", () => {- expect(- evaluateAgentHintRules(- [rule("r", [{ signal: "s", op: "eq", value: "a" }], "fired")],- signals({ s: ["a"] }),- ),- ).toEqual([]);- });-- it("requires every condition and treats an empty condition list as always", () => {- const rules = [- rule(- "both",- [- { signal: "a", op: "eq", value: true },- { signal: "b", op: "gt", value: 1 },- ],- "both",- ),- rule("always", [], "always"),- ];- expect(evaluateAgentHintRules(rules, signals({ a: true, b: 2 }))).toEqual([- "both",- "always",- ]);- expect(evaluateAgentHintRules(rules, signals({ a: true, b: 1 }))).toEqual([- "always",- ]);- });-- it("fires only the first matching rule of a group, in rule order", () => {- const rules = [- rule("independent", [], "independent"),- rule("first", [{ signal: "n", op: "gt", value: 5 }], "first", "g"),- rule("second", [{ signal: "n", op: "gt", value: 1 }], "second", "g"),- rule("third", [], "third", "g"),- ];- expect(evaluateAgentHintRules(rules, signals({ n: 9 }))).toEqual([- "independent",- "first",- ]);- expect(evaluateAgentHintRules(rules, signals({ n: 2 }))).toEqual([- "independent",- "second",- ]);- expect(evaluateAgentHintRules(rules, signals({ n: 0 }))).toEqual([- "independent",- "third",- ]);- });-- it("claims the group even when the matching rule cannot render", () => {- const rules = [- rule("needs-id", [], "Use {id}.", "g"),- rule("fallback", [], "Fallback.", "g"),- ];- expect(evaluateAgentHintRules(rules, signals({}))).toEqual([]);- expect(evaluateAgentHintRules(rules, signals({ id: "x" }))).toEqual([- "Use x.",- ]);- });-- it("fills placeholders and leaves JSON braces alone", () => {- expect(- evaluateAgentHintRules(- [- rule(- "r",- [],- 'Got {status} at {origin}; try {"url":"{origin}","n":{count}}.',- ),- ],- signals({ status: 404, origin: "https://a.example", count: 3 }),- ),- ).toEqual([- 'Got 404 at https://a.example; try {"url":"https://a.example","n":3}.',- ]);- });-- it("renders list placeholders with first and remaining modifiers", () => {- const list = signals({ items: ["#1", "#2", "#3", "#4", "#5"] });- expect(- evaluateAgentHintRules(- [- rule(- "r",- [],- "{items:first=3} and {items:remaining=3} more; all: {items}",- ),- ],- list,- ),- ).toEqual(["#1, #2, #3 and 2 more; all: #1, #2, #3, #4, #5"]);- expect(- evaluateAgentHintRules(- [rule("r", [], "{items:remaining=9}")],- signals({ items: ["#1"] }),- ),- ).toEqual(["0"]);- });-- it("emits nothing for list counts above the limit, however many digits", () => {- const list = signals({ items: ["#1"] });- for (const text of ["{items:first=101}", "{items:remaining=1000}"]) {- expect(evaluateAgentHintRules([rule("r", [], text)], list)).toEqual([]);- }- expect(- evaluateAgentHintRules([rule("r", [], "{items:first=100}")], list),- ).toEqual(["#1"]);- });-- it("emits nothing for modifiers on scalar signals", () => {- expect(- evaluateAgentHintRules([rule("r", [], "{n:first=2}")], signals({ n: 5 })),- ).toEqual([]);- });-- it("drops duplicate texts and caps the number of hints", () => {- const rules = ["a", "a", "b", "c", "d"].map((text, i) =>- rule(`r${i}`, [], text),- );- expect(evaluateAgentHintRules(rules, signals({}))).toEqual(["a", "b", "c"]);- });-});--describe("agent hint rule parsing", () => {- it("accepts a valid rule set in order", () => {- const raw = Array.from({ length: 32 }, (_, i) => ({- id: `r${i}`,- when: [],- text: `Rule ${i}.`,- }));- const rules = parseAgentHintRules(raw);- expect(rules).toHaveLength(32);- expect(rules![0]).toEqual({ id: "r0", when: [], text: "Rule 0." });- });-- it("rejects more than 32 rules", () => {- const raw = Array.from({ length: 33 }, (_, i) => ({- id: `r${i}`,- when: [],- text: "x",- }));- expect(parseAgentHintRules(raw)).toBeUndefined();- });-- it("rejects anything but an array", () => {- expect(parseAgentHintRules(undefined)).toBeUndefined();- expect(parseAgentHintRules({ id: "r" })).toBeUndefined();- });-- it.each([- [- "too many conditions",- Array(17).fill({ signal: "s", op: "eq", value: 1 }),- ],- [- "an oversized in list",- [{ signal: "s", op: "in", value: Array(33).fill(1) }],- ],- ["a non-finite number", [{ signal: "s", op: "gt", value: NaN }]],- ["an unknown operator", [{ signal: "s", op: "matches", value: "x" }]],- ])("rejects the whole set when one rule has %s", (_name, when) => {- expect(- parseAgentHintRules([- { id: "ok", when: [], text: "x" },- { id: "bad", when, text: "x" },- ]),- ).toBeUndefined();- });-- it("accepts 16 conditions", () => {- expect(- parseAgentHintRules([- {- id: "ok",- when: Array(16).fill({ signal: "s", op: "eq", value: 1 }),- text: "x",- },- ]),- ).toHaveLength(1);- });-});diff --git a/apps/api/src/lib/agent-hint-rules.ts b/apps/api/src/lib/agent-hint-rules.tsdeleted file mode 100644index b0f3881aa..000000000--- a/apps/api/src/lib/agent-hint-rules.ts+++ /dev/null@@ -1,225 +0,0 @@-import type {- AgentHintSignals,- AgentHintSignalValue,-} from "./agent-hint-signals";--/**- * Provider-supplied response guidance rules, evaluated locally against- * computed signals.- *- * - Rules are evaluated in order; a rule fires when every condition in `when`- * holds (an empty `when` always holds).- * - Rules sharing a `group` are alternatives: the first rule in the group whose- * conditions hold is the group's rule, and later rules in that group are- * skipped.- * - `text` may contain `{signal}` placeholders. List signals also accept- * `{signal:first=N}` (first N items) and `{signal:remaining=N}` (count of- * items after the first N), with N at most 100. Lists render joined with- * ", ". A rule whose text references an absent signal, or uses a modifier it- * cannot apply, emits nothing.- * - A condition on an absent signal holds only for `exists` with `false`.- */-type AgentHintRuleScalar = string | number | boolean;-type AgentHintRuleOp =- | "eq"- | "ne"- | "lt"- | "lte"- | "gt"- | "gte"- | "in"- | "exists";-type AgentHintRuleCondition =- | { signal: string; op: "eq" | "ne"; value: AgentHintRuleScalar }- | { signal: string; op: "lt" | "lte" | "gt" | "gte"; value: number }- | { signal: string; op: "in"; value: AgentHintRuleScalar[] }- | { signal: string; op: "exists"; value: boolean };-export type AgentHintRule = {- id: string;- group?: string;- when: AgentHintRuleCondition[];- text: string;-};--const MAX_RULES = 32;-const MAX_CONDITIONS = 16;-const MAX_IN_VALUES = 32;-const MAX_NAME_LENGTH = 64;-const MAX_STRING_VALUE_LENGTH = 200;-const MAX_RULE_TEXT_LENGTH = 500;-const MAX_LIST_ARGUMENT = 100;-const MAX_RULE_HINTS = 3;--const SIGNAL_NAME = /^[a-z][a-z0-9_]*$/;-const PLACEHOLDER = /\{([a-z][a-z0-9_]*)(?::(first|remaining)=(\d+))?\}/g;--function cleanString(value: unknown, maxLength: number): string | undefined {- if (typeof value !== "string") return undefined;- const cleaned = value.replace(/\p{Cc}/gu, " ").trim();- return cleaned && cleaned.length <= maxLength ? cleaned : undefined;-}--function isScalar(value: unknown): value is AgentHintRuleScalar {- return (- typeof value === "boolean" ||- (typeof value === "number" && Number.isFinite(value)) ||- (typeof value === "string" && value.length <= MAX_STRING_VALUE_LENGTH)- );-}--function parseCondition(raw: unknown): AgentHintRuleCondition | undefined {- if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined;- const { signal, op, value } = raw as Record<string, unknown>;- if (- typeof signal !== "string" ||- signal.length > MAX_NAME_LENGTH ||- !SIGNAL_NAME.test(signal)- ) {- return undefined;- }- switch (op) {- case "eq":- case "ne":- return isScalar(value) ? { signal, op, value } : undefined;- case "lt":- case "lte":- case "gt":- case "gte":- return typeof value === "number" && Number.isFinite(value)- ? { signal, op, value }- : undefined;- case "in":- return Array.isArray(value) &&- value.length <= MAX_IN_VALUES &&- value.every(isScalar)- ? { signal, op, value }- : undefined;- case "exists":- return typeof value === "boolean" ? { signal, op, value } : undefined;- default:- return undefined;- }-}--function parseRule(raw: unknown): AgentHintRule | undefined {- if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined;- const rule = raw as Record<string, unknown>;- const id = cleanString(rule.id, MAX_NAME_LENGTH);- const text = cleanString(rule.text, MAX_RULE_TEXT_LENGTH);- const group =- rule.group === undefined- ? undefined- : cleanString(rule.group, MAX_NAME_LENGTH);- const rawWhen = rule.when === undefined ? [] : rule.when;- if (- !id ||- !text ||- (rule.group !== undefined && !group) ||- !Array.isArray(rawWhen) ||- rawWhen.length > MAX_CONDITIONS- ) {- return undefined;- }- const when: AgentHintRuleCondition[] = [];- for (const value of rawWhen) {- const condition = parseCondition(value);- if (!condition) return undefined;- when.push(condition);- }- return group ? { id, group, when, text } : { id, when, text };-}--/**- * Validates a rule set as a whole: returns undefined unless it is an array of- * at most MAX_RULES rules that are all valid. Dropping individual rules could- * change which rule in a group applies, so a partially valid set is rejected.- */-export function parseAgentHintRules(- values: unknown,-): AgentHintRule[] | undefined {- if (!Array.isArray(values) || values.length > MAX_RULES) return undefined;- const rules: AgentHintRule[] = [];- for (const value of values) {- const rule = parseRule(value);- if (!rule) return undefined;- rules.push(rule);- }- return rules;-}--function conditionHolds(- condition: AgentHintRuleCondition,- signals: AgentHintSignals,-): boolean {- const actual: AgentHintSignalValue | undefined = signals.get(- condition.signal,- );- if (condition.op === "exists") {- return (actual !== undefined) === condition.value;- }- if (actual === undefined || Array.isArray(actual)) return false;- switch (condition.op) {- case "eq":- return actual === condition.value;- case "ne":- return actual !== condition.value;- case "in":- return condition.value.includes(actual);- case "lt":- return typeof actual === "number" && actual < condition.value;- case "lte":- return typeof actual === "number" && actual <= condition.value;- case "gt":- return typeof actual === "number" && actual > condition.value;- case "gte":- return typeof actual === "number" && actual >= condition.value;- }-}--function renderText(- template: string,- signals: AgentHintSignals,-): string | undefined {- let complete = true;- const rendered = template.replace(- PLACEHOLDER,- (_match, name: string, modifier?: string, count?: string) => {- const value = signals.get(name);- const n = count === undefined ? undefined : Number(count);- if (- value === undefined ||- (modifier !== undefined && !Array.isArray(value)) ||- (n !== undefined && n > MAX_LIST_ARGUMENT)- ) {- complete = false;- return "";- }- if (!Array.isArray(value)) return String(value);- if (modifier === "first") return value.slice(0, n).join(", ");- if (modifier === "remaining") {- return String(Math.max(0, value.length - (n ?? 0)));- }- return value.join(", ");- },- );- return complete ? rendered : undefined;-}--export function evaluateAgentHintRules(- rules: AgentHintRule[],- signals: AgentHintSignals,-): string[] {- const claimedGroups = new Set<string>();- const hints: string[] = [];- for (const rule of rules) {- if (hints.length >= MAX_RULE_HINTS) break;- if (rule.group !== undefined && claimedGroups.has(rule.group)) continue;- if (!rule.when.every(condition => conditionHolds(condition, signals))) {- continue;- }- if (rule.group !== undefined) claimedGroups.add(rule.group);- const text = renderText(rule.text, signals);- if (text && !hints.includes(text)) hints.push(text);- }- return hints;-}diff --git a/apps/api/src/lib/agent-hint-signals.test.ts b/apps/api/src/lib/agent-hint-signals.test.tsdeleted file mode 100644index 946444b88..000000000--- a/apps/api/src/lib/agent-hint-signals.test.ts+++ /dev/null@@ -1,211 +0,0 @@-import {- computeAgentHintSignals,- type AgentHintSignalContext,-} from "./agent-hint-signals";--const signalsFor = (overrides: Partial<AgentHintSignalContext>) =>- Object.fromEntries(- computeAgentHintSignals({- endpoint: "search",- response: { success: true, data: {} },- ...overrides,- }),- );--describe("agent hint signals", () => {- it("always reports the endpoint, outcome and capabilities", () => {- expect(signalsFor({})).toEqual({- endpoint: "search",- success: true,- can_use_map_and_crawl: false,- can_use_interact: false,- });- expect(- signalsFor({- endpoint: "map",- response: { success: false, error: "failed" },- canUseMapAndCrawl: true,- canUseInteract: true,- remainingCredits: 42,- }),- ).toEqual({- endpoint: "map",- success: false,- can_use_map_and_crawl: true,- can_use_interact: true,- remaining_credits: 42,- });- });-- it("omits non-finite remaining credits", () => {- expect(signalsFor({ remainingCredits: Infinity })).not.toHaveProperty(- "remaining_credits",- );- });-- it("describes the scraped page from metadata", () => {- expect(- signalsFor({- endpoint: "scrape",- response: {- success: true,- data: {- metadata: {- statusCode: 410,- scrapeId: "id with space",- sourceURL: "https://old.example/a",- url: "https://new.example/docs/getting-started.html",- },- },- },- }),- ).toMatchObject({- page_status: 410,- scrape_id: "id%20with%20space",- page_redirect_from: '"https://old.example/a"',- page_redirect_to: '"https://new.example/docs/getting-started.html"',- page_host: "new.example",- page_path_words: "docs getting started",- });- });-- it("falls back to the top-level scrape ID", () => {- expect(- signalsFor({- endpoint: "scrape",- response: { success: true, scrape_id: "top", data: { metadata: {} } },- }),- ).toMatchObject({ scrape_id: "top" });- });-- it("omits redirect values when the URLs match or are too long to quote", () => {- const page = (sourceURL: string, url: string) =>- signalsFor({- endpoint: "scrape",- response: { success: true, data: { metadata: { sourceURL, url } } },- });- expect(- page("https://a.example/x", "https://a.example/x"),- ).not.toHaveProperty("page_redirect_from");- expect(- page("https://a.example/x", `https://b.example/${"y".repeat(300)}`),- ).not.toHaveProperty("page_redirect_from");- });-- it("omits path words when only opaque ids remain", () => {- const result = signalsFor({- endpoint: "scrape",- response: {- success: true,- data: { metadata: { url: "https://x.example/12345" } },- },- });- expect(result).toMatchObject({ page_host: "x.example" });- expect(result).not.toHaveProperty("page_path_words");- });-- it("reports the engine and quoted query for a search results page", () => {- const pageOf = (sourceURL: string) =>- signalsFor({- endpoint: "scrape",- response: { success: true, data: { metadata: { sourceURL } } },- });- expect(pageOf("https://www.bing.com/search?q=web+scraping")).toMatchObject({- page_serp_engine: "Bing",- page_serp_query: '"web scraping"',- });- expect(pageOf("https://www.bing.com/maps?q=paris")).not.toHaveProperty(- "page_serp_engine",- );- // A results page that redirected to a consent page still was one.- expect(- signalsFor({- endpoint: "scrape",- response: {- success: true,- data: {- metadata: {- sourceURL: "https://www.google.com/search?q=firecrawl",- url: "https://consent.google.com/m?continue=x",- },- },- },- }),- ).toMatchObject({- page_serp_engine: "Google",- page_serp_query: '"firecrawl"',- });- });-- it("reports document page counts with the requestable remainder", () => {- expect(- signalsFor({- endpoint: "scrape",- response: {- success: true,- data: { metadata: { numPages: 5, totalPages: 12000 } },- },- }),- ).toMatchObject({- document_pages_returned: 5,- document_pages_total: 12000,- document_max_pages: 10000,- document_pages_requestable: 9995,- });- });-- it("never reports a negative requestable page count", () => {- expect(- signalsFor({- endpoint: "scrape",- response: {- success: true,- data: { metadata: { numPages: 12, totalPages: 5 } },- },- }),- ).toMatchObject({ document_pages_requestable: 0 });- });-- it("summarises web results, excerpts and origins", () => {- expect(- signalsFor({- response: {- success: true,- data: {- web: [- { url: "https://docs.example.com/a", markdown: "a" },- { url: "https://docs.example.com/b", position: 7 },- { url: "https://docs.example.com/c", html: "" },- { url: "javascript:alert(1)" },- { description: "no url" },- ],- },- },- }),- ).toMatchObject({- result_count: 5,- excerpt_count: 2,- excerpt_share: 0.4,- excerpt_results: ['#7 "https://docs.example.com/b"', "#4"],- origin_result_count: 3,- top_origin: "https://docs.example.com",- top_origin_count: 3,- top_origin_share: 1,- });- });-- it("reads results from a top-level data array and reports empty sets", () => {- expect(signalsFor({ response: { success: true, data: [] } })).toMatchObject(- { result_count: 0, excerpt_count: 0, origin_result_count: 0 },- );- expect(- signalsFor({ response: { success: true, data: { web: [] } } }),- ).not.toHaveProperty("excerpt_share");- });-- it("reports no result signals without a web result collection", () => {- expect(- signalsFor({ response: { success: true, data: { images: [] } } }),- ).not.toHaveProperty("result_count");- });-});diff --git a/apps/api/src/lib/agent-hint-signals.ts b/apps/api/src/lib/agent-hint-signals.tsdeleted file mode 100644index c4cdabb5a..000000000--- a/apps/api/src/lib/agent-hint-signals.ts+++ /dev/null@@ -1,249 +0,0 @@-import { detectSerpPage } from "./serp-url";--/**- * Facts about a hint-enabled response, computed locally from the response- * body and request state already in memory. Provider-supplied rules are- * matched against these values and may reference them as text placeholders;- * the values themselves never leave the API.- */-export type AgentHintEndpoint = "search" | "scrape" | "parse" | "map";--export interface AgentHintSignalContext {- endpoint: AgentHintEndpoint;- response: unknown;- remainingCredits?: number;- canUseMapAndCrawl?: boolean;- canUseInteract?: boolean;- /** Alexandria providers covering the page's host, when that lookup succeeded in time; undefined when it was skipped, late or failed. */- domainProviders?: string[];-}--export type AgentHintSignalValue = string | number | boolean | string[];-export type AgentHintSignals = Map<string, AgentHintSignalValue>;--const PDF_MAX_PAGES_LIMIT = 10000;-const HINT_URL_MAX_CHARS = 200;-const QUERY_PATH_SEGMENTS = 2;--type ObjectValue = Record<string, unknown>;--function object(value: unknown): ObjectValue {- return value !== null && typeof value === "object" && !Array.isArray(value)- ? (value as ObjectValue)- : {};-}--function httpUrl(value: unknown): URL | undefined {- if (typeof value !== "string" || !value) return undefined;- try {- const parsed = new URL(value);- return parsed.protocol === "http:" || parsed.protocol === "https:"- ? parsed- : undefined;- } catch {- return undefined;- }-}--/**- * Result URLs come from third-party pages, so they are rendered only as a- * parsed http(s) href (whitespace and quotes percent-encoded), length-capped,- * and JSON-quoted to mark them as data rather than instruction text.- */-function quotedUrl(url: URL): string | undefined {- const href = url.href;- return href.length <= HINT_URL_MAX_CHARS ? JSON.stringify(href) : undefined;-}--function resultPosition(item: ObjectValue, index: number): number {- const position = item.position;- return typeof position === "number" &&- Number.isInteger(position) &&- position > 0- ? position- : index + 1;-}--/**- * Identifier-like path segments carry no searchable meaning: numbers, hex or- * UUID-style ids, ULIDs, and long mixed alphanumeric tokens such as- * "W020260806515694454560".- */-function isOpaqueId(segment: string): boolean {- if (/^\d+$/.test(segment)) return true;- if (/^[0-9a-f-]{16,}$/i.test(segment)) return true;- if (/^[0-9A-HJKMNP-TV-Z]{26}$/i.test(segment)) return true;- return (- segment.length >= 12 &&- /^[A-Za-z0-9_]+$/.test(segment) &&- (segment.match(/\d/g)?.length ?? 0) >= 4- );-}--/** Words from the last path segments, e.g. /payments/checkout/migration-from-legacy -> "checkout migration from legacy". */-function pathWords(url: URL): string {- const segments = url.pathname- .split("/")- .map(segment => {- try {- return decodeURIComponent(segment);- } catch {- return segment;- }- })- .map(segment => segment.replace(/\.[a-z0-9]{1,5}$/i, ""))- .filter(segment => segment && !isOpaqueId(segment));- return segments- .slice(-QUERY_PATH_SEGMENTS)- .join(" ")- .replace(/[-_+.]+/g, " ")- .replace(/[^\p{L}\p{N} ]+/gu, " ")- .replace(/\s+/g, " ")- .trim()- .slice(0, 80);-}--function finiteNumber(value: unknown): number | undefined {- return typeof value === "number" && Number.isFinite(value)- ? value- : undefined;-}--function setDefined(- signals: AgentHintSignals,- name: string,- value: AgentHintSignalValue | undefined,-): void {- if (value !== undefined) signals.set(name, value);-}--function addPageSignals(- signals: AgentHintSignals,- response: ObjectValue,- metadata: ObjectValue,-): void {- if (typeof metadata.statusCode === "number") {- signals.set("page_status", metadata.statusCode);- }-- const scrapeId =- typeof metadata.scrapeId === "string" && metadata.scrapeId- ? metadata.scrapeId- : typeof response.scrape_id === "string" && response.scrape_id- ? response.scrape_id- : undefined;- if (scrapeId) signals.set("scrape_id", encodeURIComponent(scrapeId));-- const source = httpUrl(metadata.sourceURL);- const final = httpUrl(metadata.url);- if (source && final && source.href !== final.href) {- const from = quotedUrl(source);- const to = quotedUrl(final);- if (from && to) {- signals.set("page_redirect_from", from);- signals.set("page_redirect_to", to);- }- }- const serp = detectSerpPage((source ?? final)?.href ?? "");- if (serp) {- signals.set("page_serp_engine", serp.engine);- signals.set("page_serp_query", JSON.stringify(serp.query));- }- const page = final ?? source;- if (page) {- signals.set("page_host", page.hostname);- const words = pathWords(page);- if (words) signals.set("page_path_words", words);- }-- const returned = finiteNumber(metadata.numPages);- const total = finiteNumber(metadata.totalPages);- setDefined(signals, "document_pages_returned", returned);- setDefined(signals, "document_pages_total", total);- if (returned !== undefined && total !== undefined) {- const maxPages = Math.min(total, PDF_MAX_PAGES_LIMIT);- signals.set("document_max_pages", maxPages);- signals.set("document_pages_requestable", Math.max(0, maxPages - returned));- }-}--function addResultSignals(signals: AgentHintSignals, web: unknown[]): void {- signals.set("result_count", web.length);-- const excerpts: string[] = [];- const originCounts = new Map<string, number>();- web.forEach((value, index) => {- const item = object(value);- const url = typeof item.url === "string" ? item.url : undefined;- if (url === undefined) return;- const parsed = httpUrl(url);- if (- item.markdown === undefined &&- item.html === undefined &&- item.rawHtml === undefined- ) {- const quoted = parsed ? quotedUrl(parsed) : undefined;- const position = resultPosition(item, index);- excerpts.push(quoted ? `#${position} ${quoted}` : `#${position}`);- }- if (parsed) {- originCounts.set(- parsed.origin,- (originCounts.get(parsed.origin) ?? 0) + 1,- );- }- });-- signals.set("excerpt_count", excerpts.length);- signals.set("excerpt_results", excerpts);- if (web.length > 0) {- signals.set("excerpt_share", excerpts.length / web.length);- }-- const originResults = [...originCounts.values()].reduce(- (total, count) => total + count,- 0,- );- signals.set("origin_result_count", originResults);- const top = [...originCounts.entries()].sort((a, b) => b[1] - a[1])[0];- if (top) {- signals.set("top_origin", top[0]);- signals.set("top_origin_count", top[1]);- signals.set("top_origin_share", top[1] / originResults);- }-}--export function computeAgentHintSignals(- context: AgentHintSignalContext,-): AgentHintSignals {- const response = object(context.response);- const data = object(response.data);- const signals: AgentHintSignals = new Map();-- signals.set("endpoint", context.endpoint);- signals.set("success", response.success === true);- signals.set("can_use_map_and_crawl", context.canUseMapAndCrawl === true);- signals.set("can_use_interact", context.canUseInteract === true);- setDefined(- signals,- "remaining_credits",- finiteNumber(context.remainingCredits),- );-- addPageSignals(signals, response, object(data.metadata));-- if (context.domainProviders) {- signals.set("alexandria_provider_count", context.domainProviders.length);- signals.set("alexandria_providers", context.domainProviders);- setDefined(signals, "alexandria_provider", context.domainProviders[0]);- }-- const web = Array.isArray(response.data)- ? response.data- : Array.isArray(data.web)- ? data.web- : undefined;- if (web) addResultSignals(signals, web);-- return signals;-}diff --git a/apps/api/src/lib/agent-hints-provider.test.ts b/apps/api/src/lib/agent-hints-provider.test.tsdeleted file mode 100644index c2ad7d7f4..000000000--- a/apps/api/src/lib/agent-hints-provider.test.ts+++ /dev/null@@ -1,540 +0,0 @@-import { createHash, createHmac } from "node:crypto";-import http from "node:http";-import type { AddressInfo } from "node:net";-import { config } from "../config";-import {- agentHintsProviderRequestsTotal,- getProviderHints,- mergeAgentHints,- providerTeamId,- type AgentHintsProviderContext,-} from "./agent-hints-provider";--type Handler = (- body: any,- req: http.IncomingMessage,- res: http.ServerResponse,-) => void;--let handler: Handler;-let requests: { body: any; headers: http.IncomingHttpHeaders }[] = [];-let server: http.Server;-let providerUrl: string;-let teamCounter = 0;--function context(- overrides: Partial<AgentHintsProviderContext> = {},-): AgentHintsProviderContext {- return {- teamId: `team-${++teamCounter}`,- orgId: "org-1",- apiKeyId: 42,- endpoint: "scrape",- surface: "mcp",- keyless: false,- ...overrides,- };-}--function reply(status: number, payload: unknown): Handler {- return (_body, _req, res) => {- res.writeHead(status, { "content-type": "application/json" });- res.end(typeof payload === "string" ? payload : JSON.stringify(payload));- };-}--async function counter(outcome: string): Promise<number> {- const metric = await agentHintsProviderRequestsTotal.get();- return metric.values.find(v => v.labels.outcome === outcome)?.value ?? 0;-}--async function settle(holder: { settled: boolean } | undefined) {- await vi.waitFor(() => expect(holder?.settled).toBe(true));-}--describe("external agent hints provider", () => {- const original = {- url: config.AGENT_HINTS_PROVIDER_URL,- secret: config.AGENT_HINTS_PROVIDER_SECRET,- pseudonymKey: config.AGENT_HINTS_PROVIDER_PSEUDONYM_KEY,- timeout: config.AGENT_HINTS_PROVIDER_TIMEOUT_MS,- };-- beforeAll(async () => {- server = http.createServer((req, res) => {- let raw = "";- req.on("data", chunk => (raw += chunk));- req.on("end", () => {- const body = JSON.parse(raw);- requests.push({ body, headers: req.headers });- handler(body, req, res);- });- });- await new Promise<void>(resolve => server.listen(0, "127.0.0.1", resolve));- providerUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/hints`;- });-- afterAll(async () => {- server.closeAllConnections();- await new Promise(resolve => server.close(resolve));- config.AGENT_HINTS_PROVIDER_URL = original.url;- config.AGENT_HINTS_PROVIDER_SECRET = original.secret;- config.AGENT_HINTS_PROVIDER_PSEUDONYM_KEY = original.pseudonymKey;- config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = original.timeout;- });-- beforeEach(() => {- requests = [];- handler = reply(200, {- hints: [{ id: "h1", text: "Provider hint." }],- ttl_seconds: 60,- });- config.AGENT_HINTS_PROVIDER_URL = providerUrl;- config.AGENT_HINTS_PROVIDER_SECRET = undefined;- config.AGENT_HINTS_PROVIDER_PSEUDONYM_KEY = undefined;- config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = 500;- });-- afterEach(() => {- vi.restoreAllMocks();- });-- it("is inert when no provider URL is configured", async () => {- config.AGENT_HINTS_PROVIDER_URL = undefined;- const fetchSpy = vi.spyOn(globalThis, "fetch");- const disabledBefore = await counter("disabled");- expect(getProviderHints(context())).toBeUndefined();- expect(fetchSpy).not.toHaveBeenCalled();- expect(await counter("disabled")).toBe(disabledBefore + 1);- });-- it("returns an unsettled holder immediately and fills it once the provider answers", async () => {- const holder = getProviderHints(context());- expect(holder).toEqual({- settled: false,- hints: [],- rules: [],- ready: expect.any(Promise),- startedAt: expect.any(Number),- });- await settle(holder);- expect(holder!.hints).toEqual([{ id: "h1", text: "Provider hint." }]);- });-- it("resolves ready only once the holder is settled, also on failure", async () => {- const holder = getProviderHints(context());- await holder!.ready;- expect(holder!.settled).toBe(true);- expect(holder!.hints).toEqual([{ id: "h1", text: "Provider hint." }]);-- handler = reply(500, "nope");- const failed = getProviderHints(context());- await failed!.ready;- expect(failed).toMatchObject({ settled: true, hints: [], rules: [] });- });-- it("sends exactly the contract fields and never the API key", async () => {- config.AGENT_HINTS_PROVIDER_SECRET = "shared-secret";- await settle(- getProviderHints(- context({ teamId: "team-contract", surface: "cli", keyless: true }),- ),- );- expect(requests).toHaveLength(1);- const { body, headers } = requests[0];- expect(Object.keys(body).sort()).toEqual(- [- "api_key_id",- "endpoint",- "keyless",- "org_id",- "surface",- "team_id",- "version",- ].sort(),- );- expect(body).toEqual({- version: 2,- team_id: "team-contract",- org_id: "org-1",- api_key_id: 42,- endpoint: "scrape",- surface: "cli",- keyless: true,- });- expect(JSON.stringify(body)).not.toContain('api_key"');- expect(headers.authorization).toBe("Bearer shared-secret");- });-- it.each([- {- name: "keyed by the configured pseudonym key",- pseudonymKey: "k".repeat(32),- ip: "203.0.113.8",- },- {- name: "keyed by a per-process key when none is configured",- pseudonymKey: undefined,- ip: "203.0.113.9",- },- ])(- "never sends a raw keyless team ID or IP: pseudonym $name",- async ({ pseudonymKey, ip }) => {- config.AGENT_HINTS_PROVIDER_SECRET = "shared-secret";- config.AGENT_HINTS_PROVIDER_PSEUDONYM_KEY = pseudonymKey;- const teamId = `preview_keyless_${ip}`;- await settle(- getProviderHints(- context({ teamId, orgId: null, apiKeyId: 0, keyless: true }),- ),- );- expect(requests).toHaveLength(1);- const { body } = requests[0];- expect(body.team_id).toMatch(/^keyless_[0-9a-f]{64}$/);- expect(body.keyless).toBe(true);- for (const value of Object.values(body)) {- expect(String(value)).not.toContain(ip);- expect(String(value)).not.toContain("preview_keyless_");- }- expect(JSON.stringify(body)).not.toContain(ip);- const guessable = [- createHash("sha256").update(teamId).digest("hex"),- createHmac("sha256", "shared-secret").update(teamId).digest("hex"),- ];- expect(guessable.map(d => `keyless_${d}`)).not.toContain(body.team_id);- if (pseudonymKey) {- expect(body.team_id).toBe(- `keyless_${createHmac("sha256", pseudonymKey).update(teamId).digest("hex")}`,- );- }- },- );-- it("gives a keyless team the same pseudonym on every lookup", async () => {- const teamId = "preview_keyless_198.51.100.4";- await settle(- getProviderHints(context({ teamId, endpoint: "scrape", keyless: true })),- );- await settle(- getProviderHints(context({ teamId, endpoint: "search", keyless: true })),- );- expect(requests).toHaveLength(2);- expect(requests[0].body.team_id).toBe(requests[1].body.team_id);- expect(requests[0].body.team_id).toMatch(/^keyless_[0-9a-f]{64}$/);- });-- it("sends account team IDs unchanged", () => {- expect(providerTeamId("team-uuid")).toBe("team-uuid");- });-- it("omits the authorization header when no secret is configured", async () => {- await settle(getProviderHints(context()));- expect(requests[0].headers.authorization).toBeUndefined();- });-- it("serves later requests from cache and dedupes concurrent cold requests", async () => {- const ctx = context();- const first = getProviderHints(ctx);- const concurrent = getProviderHints(ctx);- expect(concurrent).toBe(first);- await settle(first);- const hitsBefore = await counter("hit");- for (let i = 0; i < 5; i++) {- expect(getProviderHints(ctx)).toBe(first);- }- expect(requests).toHaveLength(1);- expect(await counter("hit")).toBe(hitsBefore + 5);- });-- it("keys the cache by team, endpoint, and surface", async () => {- const ctx = context();- await settle(getProviderHints(ctx));- await settle(getProviderHints({ ...ctx, endpoint: "search" }));- await settle(getProviderHints({ ...ctx, surface: "api" }));- expect(requests).toHaveLength(3);- });-- it("refetches after the provider TTL expires", async () => {- handler = reply(200, { hints: [], ttl_seconds: 5 });- const ctx = context();- const start = Date.now();- const now = vi.spyOn(Date, "now").mockReturnValue(start);- await settle(getProviderHints(ctx));- now.mockReturnValue(start + 4_000);- getProviderHints(ctx);- expect(requests).toHaveLength(1);- now.mockReturnValue(start + 6_000);- await settle(getProviderHints(ctx));- expect(requests).toHaveLength(2);- });-- it("clamps the TTL to ten minutes", async () => {- handler = reply(200, { hints: [], ttl_seconds: 86_400 });- const ctx = context();- const start = Date.now();- const now = vi.spyOn(Date, "now").mockReturnValue(start);- await settle(getProviderHints(ctx));- now.mockReturnValue(start + 601_000);- await settle(getProviderHints(ctx));- expect(requests).toHaveLength(2);- });-- it.each([- {- name: "a timeout",- outcome: "timeout",- handler: ((_b, _r, res) => {- setTimeout(- () =>- reply(200, { hints: [{ id: "late", text: "Late." }] })(_b, _r, res),- 300,- );- }) as Handler,- },- { name: "a 500", outcome: "error", handler: reply(500, { hints: [] }) },- { name: "a 204", outcome: "error", handler: reply(204, "") },- { name: "invalid JSON", outcome: "error", handler: reply(200, "not json") },- {- name: "a body without a hints array",- outcome: "error",- handler: reply(200, { hints: "Provider hint." }),- },- ])(- "fails open on $name with no hints and a 30s negative cache",- async ({ outcome, handler: failing }) => {- config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = 50;- handler = failing;- const ctx = context();- const before = await counter(outcome);- const start = Date.now();- const now = vi.spyOn(Date, "now").mockReturnValue(start);- const holder = getProviderHints(ctx);- await settle(holder);- expect(holder!.hints).toEqual([]);- expect(await counter(outcome)).toBe(before + 1);-- now.mockReturnValue(start + 29_000);- expect(getProviderHints(ctx)).toBe(holder);- now.mockReturnValue(start + 31_000);- const refetched = getProviderHints(ctx);- expect(refetched).not.toBe(holder);- await settle(refetched);- },- );-- it.each([- {- name: "declared",- handler: ((_b, _r, res) => {- res.writeHead(200, {- "content-type": "application/json",- "content-length": String(1024 * 1024),- });- res.end();- }) as Handler,- },- {- name: "streamed",- handler: ((_b, _r, res) => {- res.writeHead(200, { "content-type": "application/json" });- res.write(`{"hints":[{"id":"big","text":"${"x".repeat(80 * 1024)}`);- res.end('"}]}');- }) as Handler,- },- ])(- "fails open on an oversized $name response body",- async ({ handler: oversized }) => {- handler = oversized;- const before = await counter("error");- const holder = getProviderHints(context());- await settle(holder);- expect(holder!.hints).toEqual([]);- expect(await counter("error")).toBe(before + 1);- },- );-- it("fails open on a network error", async () => {- config.AGENT_HINTS_PROVIDER_URL = "http://127.0.0.1:1/v1/hints";- const before = await counter("error");- const holder = getProviderHints(context());- await settle(holder);- expect(holder!.hints).toEqual([]);- expect(await counter("error")).toBe(before + 1);- });-- it("sanitises provider hints and ignores unknown fields", async () => {- handler = reply(200, {- hints: [- { id: " ok ", text: " Line one\nline two\u0007 ", extra: true },- { id: "too-long", text: "x".repeat(501) },- { id: "x".repeat(65), text: "Long id." },- { id: "blank", text: " " },- { id: 7, text: "Numeric id." },- "not an object",- null,- { id: "max", text: "y".repeat(500) },- ],- ttl_seconds: "soon",- future_field: { anything: true },- });- const holder = getProviderHints(context());- await settle(holder);- expect(holder!.hints).toEqual([- { id: "ok", text: "Line one line two" },- { id: "max", text: "y".repeat(500) },- ]);- });-- it("parses a valid provider rule set and cleans rule text", async () => {- handler = reply(200, {- hints: [],- rules: [- {- id: "r1",- group: "g",- when: [- { signal: "endpoint", op: "eq", value: "scrape" },- { signal: "page_status", op: "in", value: [404, 410] },- ],- text: " Rule one {page_status}.\n",- },- { id: "no-when", text: "Always." },- ],- ttl_seconds: 60,- });- const holder = getProviderHints(context());- await settle(holder);- expect(holder!.rules).toEqual([- {- id: "r1",- group: "g",- when: [- { signal: "endpoint", op: "eq", value: "scrape" },- { signal: "page_status", op: "in", value: [404, 410] },- ],- text: "Rule one {page_status}.",- },- { id: "no-when", when: [], text: "Always." },- ]);- });-- it("treats a missing rules field as no rules", async () => {- const holder = getProviderHints(context());- await settle(holder);- expect(holder!.hints).toEqual([{ id: "h1", text: "Provider hint." }]);- expect(holder!.rules).toEqual([]);- });-- it.each([- ["a non-array rules field", { id: "r1" }],- [- "one rule with an unknown operator",- [- { id: "ok", when: [], text: "Fine." },- {- id: "bad-op",- when: [{ signal: "endpoint", op: "matches", value: "s" }],- text: "Bad.",- },- ],- ],- [- "one rule with a string threshold",- [- {- id: "bad-value",- when: [{ signal: "remaining_credits", op: "lt", value: "100" }],- text: "Bad.",- },- ],- ],- ["one rule with blank text", [{ id: "blank", when: [], text: " " }]],- ["a non-object rule", ["not a rule"]],- ])(- "treats %s as a failed lookup with a negative cache",- async (_name, rules) => {- handler = reply(200, {- hints: [{ id: "h1", text: "Provider hint." }],- rules,- ttl_seconds: 60,- });- const before = await counter("error");- const ctx = context();- const holder = getProviderHints(ctx);- await settle(holder);- expect(holder!.hints).toEqual([]);- expect(holder!.rules).toEqual([]);- expect(await counter("error")).toBe(before + 1);- expect(getProviderHints(ctx)).toBe(holder);- expect(requests).toHaveLength(1);- },- );-- it("bounds the cache and evicts the oldest entry first", async () => {- const fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation(- async () =>- new Response(JSON.stringify({ hints: [], ttl_seconds: 600 }), {- status: 200,- }),- );- const oldest = context({ teamId: "evict-oldest" });- getProviderHints(oldest);- for (let i = 0; i < 10_000; i++) {- getProviderHints(context({ teamId: `evict-${i}` }));- }- const callsBefore = fetchSpy.mock.calls.length;- getProviderHints(context({ teamId: "evict-9999" }));- expect(fetchSpy.mock.calls.length).toBe(callsBefore);- getProviderHints(oldest);- expect(fetchSpy.mock.calls.length).toBe(callsBefore + 1);- });-});--describe("mergeAgentHints", () => {- const hint = (id: string, text = `${id} text`) => ({ id, text });-- it("returns deterministic hints unchanged when there are no provider hints", () => {- const deterministic = ["a", "b"];- expect(mergeAgentHints(deterministic, [])).toEqual({- hints: ["a", "b"],- providerHintIds: [],- });- });-- it("appends provider hints after deterministic ones", () => {- expect(mergeAgentHints(["a"], [hint("p1"), hint("p2")])).toEqual({- hints: ["a", "p1 text", "p2 text"],- providerHintIds: ["p1", "p2"],- });- });-- it("caps provider hints at two", () => {- expect(mergeAgentHints([], [hint("p1"), hint("p2"), hint("p3")])).toEqual({- hints: ["p1 text", "p2 text"],- providerHintIds: ["p1", "p2"],- });- });-- it("caps the total at three without dropping deterministic hints", () => {- expect(mergeAgentHints(["a", "b"], [hint("p1"), hint("p2")])).toEqual({- hints: ["a", "b", "p1 text"],- providerHintIds: ["p1"],- });- expect(mergeAgentHints(["a", "b", "c", "d"], [hint("p1")])).toEqual({- hints: ["a", "b", "c", "d"],- providerHintIds: [],- });- });-- it("drops exact duplicates of deterministic and earlier provider hints", () => {- expect(- mergeAgentHints(- ["a"],- [hint("dup", "a"), hint("p1"), hint("p1-again", "p1 text"), hint("p2")],- ),- ).toEqual({- hints: ["a", "p1 text", "p2 text"],- providerHintIds: ["p1", "p2"],- });- });-});diff --git a/apps/api/src/lib/agent-hints-provider.ts b/apps/api/src/lib/agent-hints-provider.tsdeleted file mode 100644index a151a71f2..000000000--- a/apps/api/src/lib/agent-hints-provider.ts+++ /dev/null@@ -1,257 +0,0 @@-import { createHmac, randomBytes } from "node:crypto";-import { Counter } from "prom-client";-import { z } from "zod";-import { config } from "../config";-import { parseAgentHintRules, type AgentHintRule } from "./agent-hint-rules";-import type { AgentHintEndpoint } from "./agent-hint-signals";-import type { KeylessSignupSurface } from "./keyless-signup-link";-import { logger } from "./logger";--/**- * Optional external agent hints provider. When AGENT_HINTS_PROVIDER_URL is- * set, hint-enabled requests ask a separate HTTP service for additional- * guidance: finished hints and response guidance rules. The lookup starts- * after auth and is not awaited, with one exception: an insufficient-credits- * refusal waits for an in-flight lookup, bounded by the provider timeout,- * because it is sent before the lookup could otherwise return. Every other- * response only uses provider data that has already arrived (or is cached) by- * the time it is sent. Every failure means "no provider hints and no rules".- */-export type ProviderHint = { id: string; text: string };--/** Shared by every request for the same cache key; read-only for callers. */-export type ProviderHintsHolder = {- settled: boolean;- hints: ProviderHint[];- rules: AgentHintRule[];- /** Resolves once settled; never rejects. */- ready: Promise<void>;- /** When the lookup started (Date.now()); it settles within the provider timeout of this. */- startedAt: number;-};--export type AgentHintsProviderContext = {- teamId: string;- orgId: string | null;- apiKeyId: number | null;- endpoint: AgentHintEndpoint;- surface: KeylessSignupSurface;- keyless: boolean;-};--const DEFAULT_TTL_SECONDS = 60;-const MAX_TTL_SECONDS = 600;-const NEGATIVE_TTL_MS = 30_000;-const MAX_CACHE_ENTRIES = 10_000;-const MAX_HINT_ID_LENGTH = 64;-const MAX_HINT_TEXT_LENGTH = 500;-const MAX_STORED_HINTS = 10;-const MAX_RESPONSE_BYTES = 64 * 1024;-const MAX_PROVIDER_HINTS = 2;-const MAX_TOTAL_HINTS = 3;-const AGENT_HINTS_PROVIDER_CONTRACT_VERSION = 2;--// Fixed labels only: never put team IDs or provider-supplied values here.-export const agentHintsProviderRequestsTotal = new Counter({- name: "firecrawl_agent_hints_provider_requests_total",- help: "External agent hints provider lookups by outcome: hit (served from cache or joined an in-flight fetch), miss (fetch started), timeout, error, disabled (no provider configured); eval_error counts responses whose hint computation failed and were sent without hints",- labelNames: ["outcome"],-});--const responseSchema = z.object({- hints: z.array(z.unknown()),- rules: z.unknown().optional(),- ttl_seconds: z.number().finite().optional().catch(undefined),-});--const processPseudonymKey = randomBytes(32);--/**- * Keyless team IDs identify a client address, so the provider receives a- * pseudonym instead: HMAC-SHA256 keyed by AGENT_HINTS_PROVIDER_PSEUDONYM_KEY,- * which is never sent to the provider, or by a random per-process key.- */-export function providerTeamId(teamId: string): string {- if (!teamId.startsWith("preview_keyless_")) return teamId;- const digest = createHmac(- "sha256",- config.AGENT_HINTS_PROVIDER_PSEUDONYM_KEY ?? processPseudonymKey,- )- .update(teamId)- .digest("hex");- return `keyless_${digest}`;-}--async function readJsonBody(response: Response): Promise<unknown> {- const declared = Number(response.headers.get("content-length"));- if (declared > MAX_RESPONSE_BYTES) {- await response.body?.cancel().catch(() => {});- throw new Error("Response body too large");- }- if (!response.body) throw new Error("Missing response body");- const reader = response.body.getReader();- const chunks: Uint8Array[] = [];- let size = 0;- for (;;) {- const { done, value } = await reader.read();- if (done) break;- size += value.byteLength;- if (size > MAX_RESPONSE_BYTES) {- await reader.cancel().catch(() => {});- throw new Error("Response body too large");- }- chunks.push(value);- }- return JSON.parse(Buffer.concat(chunks).toString("utf8"));-}--type CacheEntry = { holder: ProviderHintsHolder; expiresAt: number };-const cache = new Map<string, CacheEntry>();--function sanitize(value: unknown, maxLength: number): string | undefined {- if (typeof value !== "string") return undefined;- const cleaned = value.replace(/\p{Cc}/gu, " ").trim();- return cleaned && cleaned.length <= maxLength ? cleaned : undefined;-}--function sanitizeHints(values: unknown[]): ProviderHint[] {- const hints: ProviderHint[] = [];- for (const value of values) {- if (hints.length >= MAX_STORED_HINTS) break;- if (!value || typeof value !== "object") continue;- const raw = value as Record<string, unknown>;- const id = sanitize(raw.id, MAX_HINT_ID_LENGTH);- const text = sanitize(raw.text, MAX_HINT_TEXT_LENGTH);- if (id && text) hints.push({ id, text });- }- return hints;-}--async function fetchProviderHints(- url: string,- context: AgentHintsProviderContext,-): Promise<{ hints: ProviderHint[]; rules: AgentHintRule[]; ttlMs: number }> {- const secret = config.AGENT_HINTS_PROVIDER_SECRET;- try {- const response = await fetch(url, {- method: "POST",- headers: {- "content-type": "application/json",- ...(secret ? { authorization: `Bearer ${secret}` } : {}),- },- body: JSON.stringify({- version: AGENT_HINTS_PROVIDER_CONTRACT_VERSION,- team_id: providerTeamId(context.teamId),- org_id: context.orgId,- api_key_id: context.apiKeyId,- endpoint: context.endpoint,- surface: context.surface,- keyless: context.keyless,- }),- signal: AbortSignal.timeout(config.AGENT_HINTS_PROVIDER_TIMEOUT_MS),- });- if (response.status !== 200) {- await response.body?.cancel().catch(() => {});- throw new Error(`Unexpected status ${response.status}`);- }- const parsed = responseSchema.safeParse(await readJsonBody(response));- if (!parsed.success) throw new Error("Malformed response body");- const rules =- parsed.data.rules === undefined- ? []- : parseAgentHintRules(parsed.data.rules);- if (!rules) throw new Error("Malformed rules");- const ttlSeconds = Math.min(- Math.max(parsed.data.ttl_seconds ?? DEFAULT_TTL_SECONDS, 0),- MAX_TTL_SECONDS,- );- return {- hints: sanitizeHints(parsed.data.hints),- rules,- ttlMs: ttlSeconds * 1000,- };- } catch (error) {- const timedOut = error instanceof Error && error.name === "TimeoutError";- agentHintsProviderRequestsTotal.inc({- outcome: timedOut ? "timeout" : "error",- });- logger.debug("Agent hints provider lookup failed", {- module: "agent-hints-provider",- endpoint: context.endpoint,- timedOut,- error: error instanceof Error ? error.message : String(error),- });- return { hints: [], rules: [], ttlMs: NEGATIVE_TTL_MS };- }-}--/**- * Returns the cached or in-flight lookup for this team, endpoint and surface,- * starting a fetch on a miss. Never throws and never blocks. Returns undefined- * when no provider is configured.- */-export function getProviderHints(- context: AgentHintsProviderContext,-): ProviderHintsHolder | undefined {- const url = config.AGENT_HINTS_PROVIDER_URL;- if (!url) {- agentHintsProviderRequestsTotal.inc({ outcome: "disabled" });- return undefined;- }-- const key = `${context.teamId}|${context.endpoint}|${context.surface}`;- const now = Date.now();- const cached = cache.get(key);- if (cached && cached.expiresAt > now) {- agentHintsProviderRequestsTotal.inc({ outcome: "hit" });- return cached.holder;- }-- agentHintsProviderRequestsTotal.inc({ outcome: "miss" });- cache.delete(key);- if (cache.size >= MAX_CACHE_ENTRIES) {- const oldest = cache.keys().next().value;- if (oldest !== undefined) cache.delete(oldest);- }- const holder: ProviderHintsHolder = {- settled: false,- hints: [],- rules: [],- ready: Promise.resolve(),- startedAt: now,- };- const entry: CacheEntry = { holder, expiresAt: Infinity };- cache.set(key, entry);- holder.ready = fetchProviderHints(url, context).then(- ({ hints, rules, ttlMs }) => {- holder.hints = hints;- holder.rules = rules;- holder.settled = true;- entry.expiresAt = Date.now() + ttlMs;- },- );- return holder;-}--/**- * Rule hints first and never dropped, then provider hints that are not exact- * duplicates, up to the provider and total caps.- */-export function mergeAgentHints(- ruleHints: string[],- provider: ProviderHint[],-): { hints: string[]; providerHintIds: string[] } {- const hints = [...ruleHints];- const providerHintIds: string[] = [];- for (const hint of provider) {- if (- providerHintIds.length >= MAX_PROVIDER_HINTS ||- hints.length >= MAX_TOTAL_HINTS- )- break;- if (hints.includes(hint.text)) continue;- hints.push(hint.text);- providerHintIds.push(hint.id);- }- return { hints, providerHintIds };-}diff --git a/apps/api/src/lib/alexandria-domain-providers.test.ts b/apps/api/src/lib/alexandria-domain-providers.test.tsdeleted file mode 100644index a0348fd42..000000000--- a/apps/api/src/lib/alexandria-domain-providers.test.ts+++ /dev/null@@ -1,184 +0,0 @@-import { config } from "../config";-import { exchangeRequest } from "../services/alexandria/client";-import {- alexandriaDomainLookupsTotal,- clearDomainProviderCache,- domainProvidersWithin,- lookupDomainProviders,-} from "./alexandria-domain-providers";-import { computeAgentHintSignals } from "./agent-hint-signals";--vi.mock("../services/alexandria/client", () => ({- exchangeRequest: vi.fn(),-}));--const exchange = vi.mocked(exchangeRequest);--function resolves(ids: string[], delayMs = 0) {- exchange.mockImplementation(- () =>- new Promise(resolve =>- setTimeout(- () =>- resolve({- status: 200,- body: {- skills: ids.map(id => ({ id, matchedDomains: ["example"] })),- },- }),- delayMs,- ),- ),- );-}--async function count(outcome: string): Promise<number> {- const metric = await alexandriaDomainLookupsTotal.get();- return metric.values.find(v => v.labels.outcome === outcome)?.value ?? 0;-}--describe("Alexandria domain providers for agent hints", () => {- const originalUrl = config.FIRE_EXCHANGE_URL;-- beforeEach(() => {- config.FIRE_EXCHANGE_URL = "http://exchange.test";- clearDomainProviderCache();- exchange.mockReset();- });-- afterAll(() => {- config.FIRE_EXCHANGE_URL = originalUrl;- });-- it("resolves the host only and serves later requests from the cache", async () => {- resolves(["finance-yahoo-com"]);- const first = lookupDomainProviders(- "https://Finance.Yahoo.com/quote/AAPL/?p=AAPL",- "team-1",- );- expect(await domainProvidersWithin(first!, 100)).toEqual([- "finance-yahoo-com",- ]);- expect(exchange).toHaveBeenCalledTimes(1);- expect(exchange.mock.calls[0][0]).toMatchObject({- path: "/v1/skills/resolve",- body: { urls: ["https://finance.yahoo.com/"] },- });-- const hits = await count("cache_hit");- const second = lookupDomainProviders(- "https://finance.yahoo.com/quote/MSFT/",- "team-2",- );- expect(await domainProvidersWithin(second!, 0)).toEqual([- "finance-yahoo-com",- ]);- expect(exchange).toHaveBeenCalledTimes(1);- expect(await count("cache_hit")).toBe(hits + 1);- });-- it("skips a late lookup but caches its result for the next request", async () => {- resolves(["github-com"], 150);- const late = await count("late");- const lookup = lookupDomainProviders(- "https://github.com/firecrawl/firecrawl",- "team-1",- );- expect(await domainProvidersWithin(lookup!, 20)).toBeUndefined();- expect(await count("late")).toBe(late + 1);-- const concurrent = lookupDomainProviders(- "https://github.com/other/repo",- "team-1",- );- expect(concurrent).toBe(lookup);-- await lookup;- const next = lookupDomainProviders("https://github.com/a/b", "team-1");- expect(await domainProvidersWithin(next!, 0)).toEqual(["github-com"]);- expect(exchange).toHaveBeenCalledTimes(1);- });-- it("reports a failed lookup as unknown, not as no providers, and backs off", async () => {- exchange.mockRejectedValue(new Error("connect ECONNREFUSED"));- const errors = await count("error");- const lookup = lookupDomainProviders("https://example.com/", "team-1");- expect(await domainProvidersWithin(lookup!, 100)).toBeUndefined();- expect(await count("error")).toBe(errors + 1);-- exchange.mockResolvedValue({ status: 503, body: "unavailable" });- const other = lookupDomainProviders("https://example.org/", "team-1");- expect(await domainProvidersWithin(other!, 100)).toBeUndefined();-- const late = await count("late");- const again = lookupDomainProviders("https://example.com/x", "team-1");- expect(await domainProvidersWithin(again!, 0)).toBeUndefined();- expect(await count("late")).toBe(late);- expect(exchange).toHaveBeenCalledTimes(2);- });-- it("caches a host with no matching provider", async () => {- resolves([]);- const lookup = lookupDomainProviders(- "https://www.paulgraham.com/greatwork.html",- "team-1",- );- expect(await domainProvidersWithin(lookup!, 100)).toEqual([]);- const again = lookupDomainProviders(- "https://www.paulgraham.com/hwh.html",- "team-1",- );- expect(await domainProvidersWithin(again!, 0)).toEqual([]);- expect(exchange).toHaveBeenCalledTimes(1);- });-- it("drops provider ids that are not plain identifiers", async () => {- resolves(["wikipedia", "bad id\nwith text", "wikipedia"]);- const lookup = lookupDomainProviders(- "https://en.wikipedia.org/wiki/X",- "team-1",- );- expect(await domainProvidersWithin(lookup!, 100)).toEqual(["wikipedia"]);- });-- it("does nothing without an Exchange or a usable URL", () => {- expect(- lookupDomainProviders("https://user:[email protected]/", "team-1"),- ).toBeUndefined();- expect(lookupDomainProviders("ftp://example.com/", "team-1")).toBe(- undefined,- );- expect(lookupDomainProviders(undefined, "team-1")).toBeUndefined();- config.FIRE_EXCHANGE_URL = undefined;- expect(- lookupDomainProviders("https://example.com/", "team-1"),- ).toBeUndefined();- expect(exchange).not.toHaveBeenCalled();- });-- it("exposes providers as hint signals only when the lookup finished", () => {- const response = { success: true, data: { metadata: {} } };- const matched = computeAgentHintSignals({- endpoint: "scrape",- response,- domainProviders: ["finance-yahoo-com", "sec-gov"],- });- expect(matched.get("alexandria_provider_count")).toBe(2);- expect(matched.get("alexandria_provider")).toBe("finance-yahoo-com");- expect(matched.get("alexandria_providers")).toEqual([- "finance-yahoo-com",- "sec-gov",- ]);-- const unmatched = computeAgentHintSignals({- endpoint: "scrape",- response,- domainProviders: [],- });- expect(unmatched.get("alexandria_provider_count")).toBe(0);- expect(unmatched.has("alexandria_provider")).toBe(false);-- const skipped = computeAgentHintSignals({ endpoint: "scrape", response });- expect(skipped.has("alexandria_provider_count")).toBe(false);- });-});diff --git a/apps/api/src/lib/alexandria-domain-providers.ts b/apps/api/src/lib/alexandria-domain-providers.tsdeleted file mode 100644index 11d587006..000000000--- a/apps/api/src/lib/alexandria-domain-providers.ts+++ /dev/null@@ -1,146 +0,0 @@-import { Counter } from "prom-client";-import { z } from "zod";-import { config } from "../config";-import { exchangeRequest } from "../services/alexandria/client";-import { logger } from "./logger";--/**- * Alexandria providers whose tools cover a page's host, used only for agent- * hints. A lookup starts when a request arrives and runs alongside it; callers- * wait a bounded time for the result and skip it when it is late, so a lookup- * never delays or fails the request it serves.- *- * Only the host is sent, and results are cached per host: the API always- * requests the extended catalogue, so a host resolves to the same providers- * for every team. Unmatched hosts are cached as well. Failures resolve to- * undefined, never to an empty list, so an outage is not read as "no- * provider"; they are cached briefly so an unavailable Exchange is not called- * on every request.- */-const MATCH_TTL_MS = 10 * 60_000;-const ERROR_TTL_MS = 30_000;-const LOOKUP_TIMEOUT_MS = 5_000;-const MAX_CACHED_HOSTS = 10_000;-const PROVIDER_ID = /^[a-z0-9][a-z0-9._-]{0,99}$/i;--const resolveResponseSchema = z.object({- skills: z.array(z.object({ id: z.string() })),-});--type Providers = string[] | undefined;--const cache = new Map<string, { providers: Providers; expiresAt: number }>();-const inFlight = new Map<string, Promise<Providers>>();--export const alexandriaDomainLookupsTotal = new Counter({- name: "alexandria_domain_lookups_total",- help: "Alexandria domain provider lookups for agent hints, by outcome",- labelNames: ["outcome"] as const,-});--function hostOf(url: unknown): string | undefined {- if (typeof url !== "string") return undefined;- const parsed = URL.parse(url);- if (- !parsed ||- !["http:", "https:"].includes(parsed.protocol) ||- parsed.username ||- parsed.password- )- return undefined;- return parsed.hostname.toLowerCase().replace(/\.$/, "") || undefined;-}--function remember(host: string, providers: Providers, ttlMs: number): void {- cache.delete(host);- if (cache.size >= MAX_CACHED_HOSTS) {- const oldest = cache.keys().next().value;- if (oldest !== undefined) cache.delete(oldest);- }- cache.set(host, { providers, expiresAt: Date.now() + ttlMs });-}--/**- * Starts (or reuses) the provider lookup for a URL's host. Returns undefined- * when Alexandria is not configured or the URL has no usable host. The- * returned promise never rejects: it resolves to the matching providers (an- * empty list when none match) or to undefined when the lookup failed.- */-export function lookupDomainProviders(- url: unknown,- teamId: string,-): Promise<Providers> | undefined {- if (!config.FIRE_EXCHANGE_URL) return undefined;- const host = hostOf(url);- if (!host) return undefined;-- const cached = cache.get(host);- if (cached && cached.expiresAt > Date.now()) {- alexandriaDomainLookupsTotal.inc({ outcome: "cache_hit" });- return Promise.resolve(cached.providers);- }- const pending = inFlight.get(host);- if (pending) return pending;-- alexandriaDomainLookupsTotal.inc({ outcome: "cache_miss" });- const lookup = exchangeRequest({- teamId,- path: "/v1/skills/resolve",- body: { urls: [`https://${host}/`] },- timeoutMs: LOOKUP_TIMEOUT_MS,- })- .then(result => {- if (result.status !== 200)- throw new Error(`Domain resolve returned ${result.status}`);- const providers = [- ...new Set(- resolveResponseSchema- .parse(result.body)- .skills.map(skill => skill.id)- .filter(id => PROVIDER_ID.test(id)),- ),- ];- remember(host, providers, MATCH_TTL_MS);- return providers;- })- .catch(error => {- alexandriaDomainLookupsTotal.inc({ outcome: "error" });- logger.warn("Alexandria domain lookup failed", {- module: "alexandria-domain-providers",- error: error instanceof Error ? error.message : String(error),- });- remember(host, undefined, ERROR_TTL_MS);- return undefined;- })- .finally(() => inFlight.delete(host));- inFlight.set(host, lookup);- return lookup;-}--/**- * The lookup's providers if it succeeds within `ms`; undefined when it is- * late or failed.- */-export async function domainProvidersWithin(- lookup: Promise<Providers>,- ms: number,-): Promise<Providers> {- let timer: ReturnType<typeof setTimeout> | undefined;- const late = Symbol("late");- const timeout = new Promise<typeof late>(resolve => {- timer = setTimeout(() => resolve(late), ms);- });- try {- const providers = await Promise.race([lookup, timeout]);- if (providers !== late) return providers;- alexandriaDomainLookupsTotal.inc({ outcome: "late" });- return undefined;- } finally {- clearTimeout(timer);- }-}--export function clearDomainProviderCache(): void {- cache.clear();- inFlight.clear();-}diff --git a/apps/api/src/middlewares/__tests__/agent-hints.test.ts b/apps/api/src/middlewares/__tests__/agent-hints.test.tsdeleted file mode 100644index 3a2196ac3..000000000--- a/apps/api/src/middlewares/__tests__/agent-hints.test.ts+++ /dev/null@@ -1,888 +0,0 @@-import express from "express";-import { performance } from "node:perf_hooks";-import request from "supertest";-import { config } from "../../config";-import type { AgentHintRule } from "../../lib/agent-hint-rules";-import { evaluateAgentHintRules } from "../../lib/agent-hint-rules";-import { computeAgentHintSignals } from "../../lib/agent-hint-signals";-import {- agentHintsProviderRequestsTotal,- getProviderHints,- type ProviderHintsHolder,-} from "../../lib/agent-hints-provider";-import {- agentHintsMiddleware,- agentHintsProviderMiddleware,-} from "../agent-hints";--vi.mock("../../lib/agent-hint-rules", async importOriginal => {- const actual =- await importOriginal<typeof import("../../lib/agent-hint-rules")>();- return {- ...actual,- evaluateAgentHintRules: vi.fn(actual.evaluateAgentHintRules),- };-});-vi.mock("../../lib/agent-hint-signals", async importOriginal => {- const actual =- await importOriginal<typeof import("../../lib/agent-hint-signals")>();- return {- ...actual,- computeAgentHintSignals: vi.fn(actual.computeAgentHintSignals),- };-});-vi.mock("../../lib/agent-hints-provider", async importOriginal => {- const actual =- await importOriginal<typeof import("../../lib/agent-hints-provider")>();- return { ...actual, getProviderHints: vi.fn(actual.getProviderHints) };-});--async function evalErrors(): Promise<number> {- const metric = await agentHintsProviderRequestsTotal.get();- return metric.values.find(v => v.labels.outcome === "eval_error")?.value ?? 0;-}--const SUCCESS = { signal: "success", op: "eq" as const, value: true };-const TEST_RULES: AgentHintRule[] = [- {- id: "credits",- when: [{ signal: "remaining_credits", op: "lt", value: 10 }],- text: "Credits: {remaining_credits}.",- },- {- id: "interact",- group: "next",- when: [- SUCCESS,- { signal: "page_status", op: "eq", value: 401 },- { signal: "can_use_interact", op: "eq", value: true },- ],- text: "Interact: {scrape_id}.",- },- {- id: "status",- group: "next",- when: [SUCCESS, { signal: "page_status", op: "gte", value: 400 }],- text: "Status: {page_status}.",- },- {- id: "excerpts",- group: "next",- when: [SUCCESS, { signal: "excerpt_count", op: "gt", value: 0 }],- text: "Excerpts: {excerpt_results}.",- },- {- id: "origin",- group: "next",- when: [- SUCCESS,- { signal: "can_use_map_and_crawl", op: "eq", value: true },- { signal: "top_origin_count", op: "gte", value: 2 },- ],- text: "Origin: {top_origin}.",- },-];--function appFor(- {- endpoint = "search",- body = { success: true, data: {} },- status = 200,- remainingCredits,- teamId = "account-team",- rules = TEST_RULES,- domainProviders,- } = {} as any,-) {- const app = express();- app.use(express.json());- app.post("/", agentHintsMiddleware(endpoint), (req, res) => {- (req as any).auth = { team_id: teamId };- res.locals.agentCreditsRemaining = remainingCredits;- res.locals.agentHintDomainProviders = domainProviders;- res.locals.agentHintsProvider = { settled: true, hints: [], rules };- res.status(status).json(body);- });- return app;-}--const CLUSTERED_BODY = {- success: true,- data: {- web: [- { url: "https://docs.example.com/a", markdown: "a" },- { url: "https://docs.example.com/b", markdown: "b" },- { url: "https://other.example.com/c", markdown: "c" },- ],- },-};--describe("agent hint response middleware", () => {- const originalDbAuthentication = config.USE_DB_AUTHENTICATION;-- beforeEach(() => {- config.USE_DB_AUTHENTICATION = true;- });-- afterAll(() => {- config.USE_DB_AUTHENTICATION = originalDbAuthentication;- });-- it("leaves the original envelope unchanged by default", async () => {- const body = { success: true, data: { web: [] }, warning: "existing" };- const response = await request(appFor({ body, remainingCredits: 0 }))- .post("/")- .send({});- expect(response.statusCode).toBe(200);- expect(response.body).toEqual(body);- });-- it("adds top-level metadata only when explicitly enabled", async () => {- const body = {- success: true,- data: { web: [{ url: "https://example.com", description: "excerpt" }] },- warning: "existing",- };- const response = await request(appFor({ body }))- .post("/")- .set("X-Firecrawl-Agent-Hints", "TRUE")- .send({});- expect(response.statusCode).toBe(200);- expect(response.body).toEqual({- ...body,- agent_hints: ['Excerpts: #1 "https://example.com/".'],- });- });-- it.each(["false", "1", "yes"])(- "header value %s preserves the original envelope",- async value => {- const body = {- success: true,- data: { web: [{ url: "https://example.com" }] },- };- const response = await request(appFor({ body, remainingCredits: 0 }))- .post("/")- .set("X-Firecrawl-Agent-Hints", value)- .send({});- expect(response.body).toEqual(body);- },- );-- it("serves no hints without provider rules", async () => {- const body = {- success: true,- data: { metadata: { statusCode: 404 } },- };- const response = await request(- appFor({ endpoint: "scrape", body, remainingCredits: 0, rules: [] }),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.body).toEqual(body);- });-- it("passes the scrape page status and ID to the rules", async () => {- const response = await request(- appFor({- endpoint: "scrape",- body: {- success: true,- data: { metadata: { statusCode: 401, scrapeId: "scrape-id" } },- },- }),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.body.agent_hints).toEqual(["Interact: scrape-id."]);- });-- it("passes matched domain providers to the rules and changes only agent_hints", async () => {- const body = {- success: true,- data: { markdown: "page", metadata: { statusCode: 200 } },- };- const rules: AgentHintRule[] = [- {- id: "domain-provider",- when: [- SUCCESS,- { signal: "alexandria_provider_count", op: "gt", value: 0 },- ],- text: "Provider: {alexandria_provider} of {alexandria_provider_count}.",- },- ];- const matched = await request(- appFor({- endpoint: "scrape",- body,- rules,- domainProviders: ["finance-yahoo-com", "sec-gov"],- }),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(matched.body).toEqual({- ...body,- agent_hints: ["Provider: finance-yahoo-com of 2."],- });-- for (const domainProviders of [[], undefined]) {- const response = await request(- appFor({ endpoint: "scrape", body, rules, domainProviders }),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.body).toEqual(body);- }- });-- it("reports Interact as unavailable without database authentication", async () => {- config.USE_DB_AUTHENTICATION = false;- const response = await request(- appFor({- endpoint: "scrape",- body: {- success: true,- data: { metadata: { statusCode: 401, scrapeId: "scrape-id" } },- },- }),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.body.agent_hints).toEqual(["Status: 401."]);- });-- it("reports Map and Crawl as available to account teams only", async () => {- const account = await request(appFor({ body: CLUSTERED_BODY }))- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(account.body.agent_hints).toEqual([- "Origin: https://docs.example.com.",- ]);-- const keyless = await request(- appFor({ body: CLUSTERED_BODY, teamId: "preview_keyless_203.0.113.8" }),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(keyless.statusCode).toBe(200);- expect(keyless.body).toEqual(CLUSTERED_BODY);- });-- it("does not add hints when no rule applies", async () => {- const response = await request(appFor())- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.body).not.toHaveProperty("agent_hints");- });-- it("passes remaining credits to the rules alongside result guidance", async () => {- const body = {- success: true,- data: { web: [{ url: "https://example.com", description: "excerpt" }] },- };- const response = await request(appFor({ body, remainingCredits: 9 }))- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.body.agent_hints).toEqual([- "Credits: 9.",- 'Excerpts: #1 "https://example.com/".',- ]);- });-- it("preserves a failure envelope when no rule applies", async () => {- const body = {- success: false,- error: "Bad URL",- code: "BAD_REQUEST",- details: [{ field: "url" }],- };- const response = await request(appFor({ body, status: 400 }))- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.statusCode).toBe(400);- expect(response.body).toEqual(body);- });-- it("evaluates rules on a failure envelope", async () => {- const body = {- success: false,- error: "Bad URL",- code: "BAD_REQUEST",- details: [{ field: "url" }],- };- const response = await request(- appFor({ body, status: 400, remainingCredits: 0 }),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.statusCode).toBe(400);- expect(response.body).toEqual({ ...body, agent_hints: ["Credits: 0."] });- });-});--const EXCERPT_BODY = {- success: true,- data: { web: [{ url: "https://example.com", description: "excerpt" }] },-};--type TestHolder = Omit<ProviderHintsHolder, "ready" | "startedAt"> &- Partial<Pick<ProviderHintsHolder, "ready" | "startedAt">>;--function holderAppFor(- holder: TestHolder | undefined,- body: object,- {- status = 200,- insufficientCredits = false,- remainingCredits,- }: {- status?: number;- insufficientCredits?: boolean;- remainingCredits?: number;- } = {},-) {- const app = express();- app.use(express.json());- app.post(- "/",- agentHintsMiddleware("search"),- (req, res, next) => {- (req as any).auth = { team_id: "account-team" };- if (holder) {- holder.ready ??= Promise.resolve();- holder.startedAt ??= Date.now();- }- res.locals.agentHintsProvider = holder;- if (insufficientCredits) res.locals.insufficientCredits = true;- res.locals.agentCreditsRemaining = remainingCredits;- next();- },- (_req, res) => {- res.status(status).json(body);- },- );- return app;-}--describe("agent hint response middleware with an external provider", () => {- it("appends settled provider hints after rule hints", async () => {- const response = await request(- holderAppFor(- {- settled: true,- hints: [{ id: "p1", text: "Provider hint." }],- rules: TEST_RULES,- },- EXCERPT_BODY,- ),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.body.agent_hints).toEqual([- 'Excerpts: #1 "https://example.com/".',- "Provider hint.",- ]);- });-- it("adds provider hints when no rule applies", async () => {- const body = { success: true, data: {} };- const response = await request(- holderAppFor(- {- settled: true,- hints: [{ id: "p1", text: "Provider hint." }],- rules: [],- },- body,- ),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.body).toEqual({ ...body, agent_hints: ["Provider hint."] });- });-- it("ignores provider hints that have not arrived yet", async () => {- const body = { success: true, data: {} };- const response = await request(- holderAppFor(- {- settled: false,- hints: [{ id: "p1", text: "Provider hint." }],- rules: [],- },- body,- ),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.body).toEqual(body);- });-- it("does not add provider hints to a failure envelope", async () => {- const body = { success: false, error: "Bad URL", code: "BAD_REQUEST" };- const response = await request(- holderAppFor(- {- settled: true,- hints: [{ id: "p1", text: "Provider hint." }],- rules: [],- },- body,- ),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.body).toEqual(body);- });-- it("does not add provider hints without the opt-in header", async () => {- const body = { success: true, data: {} };- const response = await request(- holderAppFor(- {- settled: true,- hints: [{ id: "p1", text: "Provider hint." }],- rules: [],- },- body,- ),- )- .post("/")- .send({});- expect(response.body).toEqual(body);- });-});--describe("agent hint failures never change the response", () => {- const RULES = [- {- id: "results",- when: [{ signal: "excerpt_count", op: "gt" as const, value: 0 }],- text: "Rule: {excerpt_count}.",- },- ];- const boom = () => {- throw new Error("boom");- };-- afterEach(() => {- vi.mocked(evaluateAgentHintRules).mockClear();- });-- it.each([- [- "the matcher",- () => vi.mocked(evaluateAgentHintRules).mockImplementationOnce(boom),- RULES,- ],- [- "signal computation",- () => vi.mocked(computeAgentHintSignals).mockImplementationOnce(boom),- RULES,- ],- ])(- "sends the original body with status 200 when %s throws",- async (_name, arm, rules) => {- arm();- const before = await evalErrors();- const response = await request(- holderAppFor({ settled: true, hints: [], rules }, EXCERPT_BODY),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.statusCode).toBe(200);- expect(response.body).toEqual(EXCERPT_BODY);- expect(await evalErrors()).toBe(before + 1);- },- );-- it("still responds when the provider lookup cannot start", async () => {- vi.mocked(getProviderHints).mockImplementationOnce(boom);- const app = express();- app.use(express.json());- app.post(- "/",- agentHintsMiddleware("search"),- (req, _res, next) => {- (req as any).auth = { team_id: "account-team" };- next();- },- agentHintsProviderMiddleware("search"),- (_req, res) => {- res.json(EXCERPT_BODY);- },- );- const response = await request(app)- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.statusCode).toBe(200);- expect(response.body).toEqual(EXCERPT_BODY);- });-});--describe("agent hints provider middleware", () => {- const original = {- url: config.AGENT_HINTS_PROVIDER_URL,- timeout: config.AGENT_HINTS_PROVIDER_TIMEOUT_MS,- };- let teamCounter = 0;-- function providerAppFor(- auth: { team_id: string; org_id?: string | null } | undefined,- acuc?: { api_key: string; api_key_id: number },- ) {- const app = express();- app.use(express.json());- app.post(- "/",- agentHintsMiddleware("scrape"),- (req, _res, next) => {- (req as any).auth = auth;- (req as any).acuc = acuc;- next();- },- agentHintsProviderMiddleware("scrape"),- (_req, res) => {- res.json({ success: true, data: {} });- },- );- return app;- }-- function mockProvider(gate?: Promise<void>) {- return vi.spyOn(globalThis, "fetch").mockImplementation(async () => {- await gate;- return new Response(- JSON.stringify({- hints: [{ id: "p1", text: "Provider hint." }],- ttl_seconds: 60,- }),- { status: 200 },- );- });- }-- beforeEach(() => {- config.AGENT_HINTS_PROVIDER_URL = "http://hints.invalid/v1/hints";- config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = 2000;- });-- afterEach(() => {- vi.restoreAllMocks();- });-- afterAll(() => {- config.AGENT_HINTS_PROVIDER_URL = original.url;- config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = original.timeout;- });-- it("leaves the response unchanged when no provider is configured", async () => {- config.AGENT_HINTS_PROVIDER_URL = undefined;- const fetchSpy = mockProvider();- const response = await request(- providerAppFor({ team_id: `team-${++teamCounter}` }),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.body).toEqual({ success: true, data: {} });- expect(fetchSpy).not.toHaveBeenCalled();- });-- it("serves no hints when the provider sends a malformed rule set", async () => {- vi.spyOn(globalThis, "fetch").mockImplementation(- async () =>- new Response(- JSON.stringify({- hints: [{ id: "p1", text: "Provider hint." }],- rules: [- { id: "ok", when: [], text: "Rule hint." },- { id: "bad", when: [{ signal: "s", op: "nope" }], text: "x" },- ],- ttl_seconds: 60,- }),- { status: 200 },- ),- );- const body = { success: true, data: { metadata: { statusCode: 404 } } };- const app = express();- app.use(express.json());- app.post(- "/",- agentHintsMiddleware("scrape"),- (req, _res, next) => {- (req as any).auth = { team_id: `team-${++teamCounter}` };- next();- },- agentHintsProviderMiddleware("scrape"),- async (_req, res) => {- await vi.waitFor(() =>- expect(res.locals.agentHintsProvider?.settled).toBe(true),- );- res.json(body);- },- );- const response = await request(app)- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(response.statusCode).toBe(200);- expect(response.body).toEqual(body);- });-- it("does not call the provider without the opt-in header", async () => {- const fetchSpy = mockProvider();- await request(providerAppFor({ team_id: `team-${++teamCounter}` }))- .post("/")- .send({});- expect(fetchSpy).not.toHaveBeenCalled();- });-- it("does not call the provider without an authenticated team", async () => {- const fetchSpy = mockProvider();- await request(providerAppFor(undefined))- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(fetchSpy).not.toHaveBeenCalled();- });-- it("sends the request context without the API key", async () => {- const fetchSpy = mockProvider();- const teamId = `team-${++teamCounter}`;- await request(- providerAppFor(- { team_id: teamId, org_id: "org-1" },- { api_key: "fc-secret-key", api_key_id: 7 },- ),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .set("x-origin", "mcp")- .send({});- expect(fetchSpy).toHaveBeenCalledTimes(1);- const body = String(fetchSpy.mock.calls[0][1]?.body);- expect(JSON.parse(body)).toEqual({- version: 2,- team_id: teamId,- org_id: "org-1",- api_key_id: 7,- endpoint: "scrape",- surface: "mcp",- keyless: false,- });- expect(body).not.toContain("fc-secret-key");- });-- it("does not send a keyless caller's team ID or IP", async () => {- const fetchSpy = mockProvider();- const ip = `192.0.2.${++teamCounter}`;- await request(- providerAppFor(- { team_id: `preview_keyless_${ip}`, org_id: null },- { api_key: "", api_key_id: 0 },- ),- )- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(fetchSpy).toHaveBeenCalledTimes(1);- const body = JSON.parse(String(fetchSpy.mock.calls[0][1]?.body));- expect(body.keyless).toBe(true);- expect(body.team_id).toMatch(/^keyless_[0-9a-f]{64}$/);- for (const value of Object.values(body)) {- expect(String(value)).not.toContain(ip);- expect(String(value)).not.toContain("preview_keyless_");- }- });-- it("does not wait for a pending provider and serves its hints from cache afterwards", async () => {- let release!: () => void;- const fetchSpy = mockProvider(- new Promise<void>(resolve => (release = resolve)),- );- const app = providerAppFor({ team_id: `team-${++teamCounter}` });-- const first = await request(app)- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(first.body).toEqual({ success: true, data: {} });- expect(fetchSpy).toHaveBeenCalledTimes(1);-- release();- await vi.waitFor(- async () => {- const second = await request(app)- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- expect(second.body.agent_hints).toEqual(["Provider hint."]);- },- { timeout: 3000, interval: 50 },- );- expect(fetchSpy).toHaveBeenCalledTimes(1);- });-});--describe("insufficient-credits refusals and a pending provider lookup", () => {- // Timing assertions use a wide margin between "waited for the bound" and- // "did not wait": a long bound (LONG) that a correct response never reaches,- // or a short one (SHORT) checked only from below, so scheduling delay under- // load cannot fail a correct response.- const LONG = 10_000;- const SHORT = 60;- const REFUSAL = { success: false, error: "Insufficient credits." };- const CREDIT_RULES: AgentHintRule[] = [- {- id: "out",- group: "credits",- when: [- { signal: "success", op: "eq", value: false },- { signal: "remaining_credits", op: "lt", value: 1 },- ],- text: "Out of credits.",- },- {- id: "low",- group: "credits",- when: [{ signal: "remaining_credits", op: "lt", value: 100 }],- text: "Low on credits.",- },- ];- const originalTimeout = config.AGENT_HINTS_PROVIDER_TIMEOUT_MS;-- beforeEach(() => {- config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = LONG;- });-- afterAll(() => {- config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = originalTimeout;- });-- /** A lookup that settles with CREDIT_RULES after `ms`, or never. */- function pendingHolder(ms?: number): TestHolder {- const holder: TestHolder = {- settled: false,- hints: [],- rules: [],- startedAt: Date.now(),- };- holder.ready = new Promise<void>(resolve => {- if (ms === undefined) return;- setTimeout(() => {- holder.rules = CREDIT_RULES;- holder.settled = true;- resolve();- }, ms);- });- return holder;- }-- async function timed(app: express.Express) {- const started = performance.now();- const response = await request(app)- .post("/")- .set("X-Firecrawl-Agent-Hints", "true")- .send({});- return { response, ms: performance.now() - started };- }-- const REFUSED = {- status: 402,- insufficientCredits: true,- remainingCredits: 0,- };-- it("waits for a lookup that settles within the bound and adds its hints", async () => {- const { response, ms } = await timed(- holderAppFor(pendingHolder(15), REFUSAL, REFUSED),- );- expect(response.statusCode).toBe(402);- expect(response.body).toEqual({- ...REFUSAL,- agent_hints: ["Out of credits."],- });- expect(ms).toBeLessThan(LONG / 2);- });-- it("sends the refusal at the bound without hints when the lookup is slower", async () => {- config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = SHORT;- const { response, ms } = await timed(- holderAppFor(pendingHolder(), REFUSAL, REFUSED),- );- expect(response.statusCode).toBe(402);- expect(response.body).toEqual(REFUSAL);- expect(ms).toBeGreaterThanOrEqual(SHORT - 5);- });-- it("counts the bound from when the lookup started", async () => {- const holder = pendingHolder();- holder.startedAt = Date.now() - (LONG - 20);- const { response, ms } = await timed(- holderAppFor(holder, REFUSAL, REFUSED),- );- expect(response.body).toEqual(REFUSAL);- expect(ms).toBeLessThan(LONG / 2);- });-- it("does not delay a successful response while the lookup is pending", async () => {- const body = { success: true, data: {} };- const { response, ms } = await timed(- holderAppFor(pendingHolder(), body, { remainingCredits: 5 }),- );- expect(response.statusCode).toBe(200);- expect(response.body).toEqual(body);- expect(ms).toBeLessThan(LONG / 2);- });-- it("does not delay other refusals while the lookup is pending", async () => {- const { response, ms } = await timed(- holderAppFor(pendingHolder(), REFUSAL, {- status: 402,- remainingCredits: 0,- }),- );- expect(response.body).toEqual(REFUSAL);- expect(ms).toBeLessThan(LONG / 2);- });-- it("does not wait when the lookup has already settled", async () => {- const { response, ms } = await timed(- holderAppFor(- {- settled: true,- hints: [],- rules: CREDIT_RULES,- ready: new Promise(() => {}),- },- REFUSAL,- REFUSED,- ),- );- expect(response.body).toEqual({- ...REFUSAL,- agent_hints: ["Out of credits."],- });- expect(ms).toBeLessThan(LONG / 2);- });-});diff --git a/apps/api/src/middlewares/agent-hints.ts b/apps/api/src/middlewares/agent-hints.tsdeleted file mode 100644index 7a93b321b..000000000--- a/apps/api/src/middlewares/agent-hints.ts+++ /dev/null@@ -1,166 +0,0 @@-import type { Request, RequestHandler, Response } from "express";-import { config } from "../config";-import type { RequestWithMaybeAuth } from "../controllers/v1/types";-import { evaluateAgentHintRules } from "../lib/agent-hint-rules";-import {- computeAgentHintSignals,- type AgentHintEndpoint,-} from "../lib/agent-hint-signals";-import {- agentHintsProviderRequestsTotal,- getProviderHints,- mergeAgentHints,- providerTeamId,- type ProviderHintsHolder,-} from "../lib/agent-hints-provider";-import { keylessSignupSurface } from "../lib/keyless-signup-link";-import { logger } from "../lib/logger";--export function agentHintsRequested(req: Request): boolean {- return req.get("X-Firecrawl-Agent-Hints")?.trim().toLowerCase() === "true";-}--function agentHintsFor(- req: Request,- res: Response,- endpoint: AgentHintEndpoint,- body: Record<string, unknown>,-): string[] {- const teamId = (req as RequestWithMaybeAuth).auth?.team_id;- const provider: ProviderHintsHolder | undefined =- res.locals.agentHintsProvider;- const context = {- endpoint,- response: body,- remainingCredits: res.locals.agentCreditsRemaining,- canUseMapAndCrawl: !!teamId && !teamId.startsWith("preview_keyless_"),- canUseInteract: config.USE_DB_AUTHENTICATION === true,- domainProviders: res.locals.agentHintDomainProviders,- };- const rules = provider?.settled ? provider.rules : [];- const { hints, providerHintIds } = mergeAgentHints(- rules.length > 0- ? evaluateAgentHintRules(rules, computeAgentHintSignals(context))- : [],- provider?.settled && body.success === true ? provider.hints : [],- );- if (providerHintIds.length > 0) {- logger.info("Served external agent hints", {- module: "agent-hints-provider",- endpoint,- team_id: teamId ? providerTeamId(teamId) : undefined,- hint_ids: providerHintIds,- });- }- return hints;-}--/**- * How long an insufficient-credits refusal may wait for the in-flight provider- * lookup: whatever is left of the provider timeout since the lookup started.- * The refusal is sent a few milliseconds after auth, so without this wait it- * would never carry rule hints on a cold cache. Every other response is sent- * as soon as the handler calls res.json.- */-function providerWaitMs(res: Response): number {- const provider: ProviderHintsHolder | undefined =- res.locals.agentHintsProvider;- if (res.locals.insufficientCredits !== true || !provider || provider.settled)- return 0;- return Math.min(- config.AGENT_HINTS_PROVIDER_TIMEOUT_MS,- provider.startedAt + config.AGENT_HINTS_PROVIDER_TIMEOUT_MS - Date.now(),- );-}--/**- * Only registered on business POST routes, never feedback or polling routes.- * Hint computation must never change the response otherwise: any error sends- * the original body unchanged.- */-export function agentHintsMiddleware(- endpoint: AgentHintEndpoint,-): RequestHandler {- return (req, res, next) => {- if (!agentHintsRequested(req)) return next();- const json = res.json;- const send = (body: Record<string, unknown>) => {- let output = body;- try {- const hints = agentHintsFor(req, res, endpoint, body);- if (hints.length > 0) output = { ...body, agent_hints: hints };- } catch (error) {- agentHintsProviderRequestsTotal.inc({ outcome: "eval_error" });- logger.warn("Agent hints computation failed", {- module: "agent-hints",- endpoint,- error: error instanceof Error ? error.message : String(error),- });- }- return json.call(res, output);- };- res.json = function (body) {- if (!body || typeof body !== "object" || Array.isArray(body))- return json.call(this, body);- const waitMs = providerWaitMs(res);- if (waitMs <= 0) return send(body);- // Deferred send: the only caller of this path returns right after- // res.json and never touches the response again, so returning before- // the body is written is safe. The wait is bounded and never rejects.- let timer: ReturnType<typeof setTimeout> | undefined;- const deadline = new Promise<void>(resolve => {- timer = setTimeout(resolve, waitMs);- });- void Promise.race([res.locals.agentHintsProvider.ready, deadline])- .then(() => {- clearTimeout(timer);- if (!res.writableEnded) send(body);- })- .catch(error => {- logger.warn("Deferred response could not be sent", {- module: "agent-hints",- endpoint,- error: error instanceof Error ? error.message : String(error),- });- });- return this;- };- next();- };-}--/**- * Starts the optional external hints lookup. Must run after authMiddleware so- * the team is known; it never waits for the provider.- */-export function agentHintsProviderMiddleware(- endpoint: AgentHintEndpoint,-): RequestHandler {- return (req, res, next) => {- if (!agentHintsRequested(req)) return next();- const authReq = req as RequestWithMaybeAuth;- const teamId = authReq.auth?.team_id;- if (!teamId) return next();- const apiKeyId = authReq.acuc?.api_key_id;- try {- res.locals.agentHintsProvider = getProviderHints({- teamId,- orgId: authReq.auth?.org_id ?? null,- apiKeyId:- typeof apiKeyId === "number" && Number.isFinite(apiKeyId)- ? apiKeyId- : null,- endpoint,- surface: keylessSignupSurface(req),- keyless: teamId.startsWith("preview_keyless_"),- });- } catch (error) {- logger.warn("Agent hints provider lookup could not start", {- module: "agent-hints-provider",- endpoint,- error: error instanceof Error ? error.message : String(error),- });- }- next();- };-}diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 4afdc771a..c45da32ea 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -199,11 +199,6 @@ export function checkCreditsMiddleware(return next();}- // Keep the authoritative balance available to opt-in response guidance.- // `req.account.remainingCredits` deliberately becomes Infinity when- // Autumn allows overage, so it cannot carry this informational signal.- res.locals.agentCreditsRemaining = autumnResult.remaining;-const success = autumnResult.allowed;// When Autumn allows the request (including overage), don't let a// small remaining balance clamp downstream limits (e.g. crawl).@@ -261,7 +256,6 @@ export function checkCreditsMiddleware(!res.headersSent &&req.auth.team_id !== "8c528896-7882-4587-a4b6-768b721b0b53") {- res.locals.insufficientCredits = true;return res.status(402).json({success: false,error:diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex c5bb5ae1e..8da444756 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -1,7 +1,3 @@-import {- agentHintsMiddleware,- agentHintsProviderMiddleware,-} from "../middlewares/agent-hints";import express from "express";import multer from "multer";import { config } from "../config";@@ -193,13 +189,11 @@ registerMcpActionLogReadRoute(v2Router.post("/search",- agentHintsMiddleware("search"),researchCategoryNoticeMiddleware,authMiddleware(RateLimiterMode.Search, {allowKeyless: true,allowAgentManagedKey: true,}),- agentHintsProviderMiddleware("search"),countryCheck,checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),blocklistMiddleware,@@ -241,9 +235,7 @@ v2Router.put(v2Router.post("/parse",- agentHintsMiddleware("parse"),authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),- agentHintsProviderMiddleware("parse"),countryCheck,checkCreditsMiddleware(1),parsePayloadMiddleware,@@ -252,12 +244,10 @@ v2Router.post(v2Router.post("/scrape",- agentHintsMiddleware("scrape"),authMiddleware(RateLimiterMode.Scrape, {allowKeyless: true,allowAgentManagedKey: true,}),- agentHintsProviderMiddleware("scrape"),countryCheck,checkCreditsMiddleware(1),scrapeBlocklistMiddleware,@@ -296,9 +286,7 @@ v2Router.post(v2Router.post("/map",- agentHintsMiddleware("map"),authMiddleware(RateLimiterMode.Map),- agentHintsProviderMiddleware("map"),checkCreditsMiddleware(1),blocklistMiddleware,wrap(mapController),