Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): let insufficient-credits responses wait briefly for in-flight hint rules (#5094)
Rakshith Ramprakash5241178e9444
diff --git a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts b/apps/api/src/__tests__/routes/check-credits-middleware.test.tsindex 54f79dbf7..724fc8746 100644--- a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts+++ b/apps/api/src/__tests__/routes/check-credits-middleware.test.ts@@ -108,6 +108,7 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {expect(res.status).not.toHaveBeenCalled();expect(req.account.remainingCredits).toBe(Infinity);expect(res.locals.agentCreditsRemaining).toBe(0);+ expect(res.locals.insufficientCredits).toBeUndefined();// request body limit must NOT have been clamped down to 0expect(req.body.limit).toBe(100);});@@ -120,6 +121,7 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {expect(res.status).toHaveBeenCalledWith(402);expect(res.locals.agentCreditsRemaining).toBe(0);+ expect(res.locals.insufficientCredits).toBe(true);expect(logger.error).toHaveBeenCalledWith(expect.any(String),expect.not.objectContaining({ request: expect.anything() }),diff --git a/apps/api/src/lib/agent-hints-provider.test.ts b/apps/api/src/lib/agent-hints-provider.test.tsindex f2f0e7553..c2ad7d7f4 100644--- a/apps/api/src/lib/agent-hints-provider.test.ts+++ b/apps/api/src/lib/agent-hints-provider.test.ts@@ -110,11 +110,29 @@ describe("external agent hints provider", () => {it("returns an unsettled holder immediately and fills it once the provider answers", async () => {const holder = getProviderHints(context());- expect(holder).toEqual({ settled: false, hints: [], rules: [] });+ expect(holder).toEqual({+ settled: false,+ hints: [],+ rules: [],+ ready: expect.any(Promise),+ startedAt: expect.any(Number),+ });await settle(holder);expect(holder!.hints).toEqual([{ id: "h1", text: "Provider hint." }]);});+ it("resolves ready only once the holder is settled, also on failure", async () => {+ const holder = getProviderHints(context());+ await holder!.ready;+ expect(holder!.settled).toBe(true);+ expect(holder!.hints).toEqual([{ id: "h1", text: "Provider hint." }]);++ handler = reply(500, "nope");+ const failed = getProviderHints(context());+ await failed!.ready;+ expect(failed).toMatchObject({ settled: true, hints: [], rules: [] });+ });+it("sends exactly the contract fields and never the API key", async () => {config.AGENT_HINTS_PROVIDER_SECRET = "shared-secret";await settle(diff --git a/apps/api/src/lib/agent-hints-provider.ts b/apps/api/src/lib/agent-hints-provider.tsindex c1a0d05de..a151a71f2 100644--- a/apps/api/src/lib/agent-hints-provider.ts+++ b/apps/api/src/lib/agent-hints-provider.ts@@ -11,9 +11,11 @@ import { logger } from "./logger";* Optional external agent hints provider. When AGENT_HINTS_PROVIDER_URL is* set, hint-enabled requests ask a separate HTTP service for additional* guidance: finished hints and response guidance rules. The lookup starts- * after auth and is never awaited: the response only uses provider data that- * has already arrived (or is cached) by the time it is sent. Every failure- * means "no provider hints and no rules".+ * after auth and is not awaited, with one exception: an insufficient-credits+ * refusal waits for an in-flight lookup, bounded by the provider timeout,+ * because it is sent before the lookup could otherwise return. Every other+ * response only uses provider data that has already arrived (or is cached) by+ * the time it is sent. Every failure means "no provider hints and no rules".*/export type ProviderHint = { id: string; text: string };@@ -22,6 +24,10 @@ export type ProviderHintsHolder = {settled: boolean;hints: ProviderHint[];rules: AgentHintRule[];+ /** Resolves once settled; never rejects. */+ ready: Promise<void>;+ /** When the lookup started (Date.now()); it settles within the provider timeout of this. */+ startedAt: number;};export type AgentHintsProviderContext = {@@ -207,18 +213,24 @@ export function getProviderHints(const oldest = cache.keys().next().value;if (oldest !== undefined) cache.delete(oldest);}- const entry: CacheEntry = {- holder: { settled: false, hints: [], rules: [] },- expiresAt: Infinity,+ const holder: ProviderHintsHolder = {+ settled: false,+ hints: [],+ rules: [],+ ready: Promise.resolve(),+ startedAt: now,};+ const entry: CacheEntry = { holder, expiresAt: Infinity };cache.set(key, entry);- void fetchProviderHints(url, context).then(({ hints, rules, ttlMs }) => {- entry.holder.hints = hints;- entry.holder.rules = rules;- entry.holder.settled = true;- entry.expiresAt = Date.now() + ttlMs;- });- return entry.holder;+ holder.ready = fetchProviderHints(url, context).then(+ ({ hints, rules, ttlMs }) => {+ holder.hints = hints;+ holder.rules = rules;+ holder.settled = true;+ entry.expiresAt = Date.now() + ttlMs;+ },+ );+ return holder;}/**diff --git a/apps/api/src/middlewares/__tests__/agent-hints.test.ts b/apps/api/src/middlewares/__tests__/agent-hints.test.tsindex 23299a8b9..3a2196ac3 100644--- a/apps/api/src/middlewares/__tests__/agent-hints.test.ts+++ b/apps/api/src/middlewares/__tests__/agent-hints.test.ts@@ -1,4 +1,5 @@import express from "express";+import { performance } from "node:perf_hooks";import request from "supertest";import { config } from "../../config";import type { AgentHintRule } from "../../lib/agent-hint-rules";@@ -337,7 +338,22 @@ const EXCERPT_BODY = {data: { web: [{ url: "https://example.com", description: "excerpt" }] },};-function holderAppFor(holder: ProviderHintsHolder | undefined, body: object) {+type TestHolder = Omit<ProviderHintsHolder, "ready" | "startedAt"> &+ Partial<Pick<ProviderHintsHolder, "ready" | "startedAt">>;++function holderAppFor(+ holder: TestHolder | undefined,+ body: object,+ {+ status = 200,+ insufficientCredits = false,+ remainingCredits,+ }: {+ status?: number;+ insufficientCredits?: boolean;+ remainingCredits?: number;+ } = {},+) {const app = express();app.use(express.json());app.post(@@ -345,11 +361,17 @@ function holderAppFor(holder: ProviderHintsHolder | undefined, body: object) {agentHintsMiddleware("search"),(req, res, next) => {(req as any).auth = { team_id: "account-team" };+ if (holder) {+ holder.ready ??= Promise.resolve();+ holder.startedAt ??= Date.now();+ }res.locals.agentHintsProvider = holder;+ if (insufficientCredits) res.locals.insufficientCredits = true;+ res.locals.agentCreditsRemaining = remainingCredits;next();},(_req, res) => {- res.json(body);+ res.status(status).json(body);},);return app;@@ -721,3 +743,146 @@ describe("agent hints provider middleware", () => {expect(fetchSpy).toHaveBeenCalledTimes(1);});});++describe("insufficient-credits refusals and a pending provider lookup", () => {+ // Timing assertions use a wide margin between "waited for the bound" and+ // "did not wait": a long bound (LONG) that a correct response never reaches,+ // or a short one (SHORT) checked only from below, so scheduling delay under+ // load cannot fail a correct response.+ const LONG = 10_000;+ const SHORT = 60;+ const REFUSAL = { success: false, error: "Insufficient credits." };+ const CREDIT_RULES: AgentHintRule[] = [+ {+ id: "out",+ group: "credits",+ when: [+ { signal: "success", op: "eq", value: false },+ { signal: "remaining_credits", op: "lt", value: 1 },+ ],+ text: "Out of credits.",+ },+ {+ id: "low",+ group: "credits",+ when: [{ signal: "remaining_credits", op: "lt", value: 100 }],+ text: "Low on credits.",+ },+ ];+ const originalTimeout = config.AGENT_HINTS_PROVIDER_TIMEOUT_MS;++ beforeEach(() => {+ config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = LONG;+ });++ afterAll(() => {+ config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = originalTimeout;+ });++ /** A lookup that settles with CREDIT_RULES after `ms`, or never. */+ function pendingHolder(ms?: number): TestHolder {+ const holder: TestHolder = {+ settled: false,+ hints: [],+ rules: [],+ startedAt: Date.now(),+ };+ holder.ready = new Promise<void>(resolve => {+ if (ms === undefined) return;+ setTimeout(() => {+ holder.rules = CREDIT_RULES;+ holder.settled = true;+ resolve();+ }, ms);+ });+ return holder;+ }++ async function timed(app: express.Express) {+ const started = performance.now();+ const response = await request(app)+ .post("/")+ .set("X-Firecrawl-Agent-Hints", "true")+ .send({});+ return { response, ms: performance.now() - started };+ }++ const REFUSED = {+ status: 402,+ insufficientCredits: true,+ remainingCredits: 0,+ };++ it("waits for a lookup that settles within the bound and adds its hints", async () => {+ const { response, ms } = await timed(+ holderAppFor(pendingHolder(15), REFUSAL, REFUSED),+ );+ expect(response.statusCode).toBe(402);+ expect(response.body).toEqual({+ ...REFUSAL,+ agent_hints: ["Out of credits."],+ });+ expect(ms).toBeLessThan(LONG / 2);+ });++ it("sends the refusal at the bound without hints when the lookup is slower", async () => {+ config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = SHORT;+ const { response, ms } = await timed(+ holderAppFor(pendingHolder(), REFUSAL, REFUSED),+ );+ expect(response.statusCode).toBe(402);+ expect(response.body).toEqual(REFUSAL);+ expect(ms).toBeGreaterThanOrEqual(SHORT - 5);+ });++ it("counts the bound from when the lookup started", async () => {+ const holder = pendingHolder();+ holder.startedAt = Date.now() - (LONG - 20);+ const { response, ms } = await timed(+ holderAppFor(holder, REFUSAL, REFUSED),+ );+ expect(response.body).toEqual(REFUSAL);+ expect(ms).toBeLessThan(LONG / 2);+ });++ it("does not delay a successful response while the lookup is pending", async () => {+ const body = { success: true, data: {} };+ const { response, ms } = await timed(+ holderAppFor(pendingHolder(), body, { remainingCredits: 5 }),+ );+ expect(response.statusCode).toBe(200);+ expect(response.body).toEqual(body);+ expect(ms).toBeLessThan(LONG / 2);+ });++ it("does not delay other refusals while the lookup is pending", async () => {+ const { response, ms } = await timed(+ holderAppFor(pendingHolder(), REFUSAL, {+ status: 402,+ remainingCredits: 0,+ }),+ );+ expect(response.body).toEqual(REFUSAL);+ expect(ms).toBeLessThan(LONG / 2);+ });++ it("does not wait when the lookup has already settled", async () => {+ const { response, ms } = await timed(+ holderAppFor(+ {+ settled: true,+ hints: [],+ rules: CREDIT_RULES,+ ready: new Promise(() => {}),+ },+ REFUSAL,+ REFUSED,+ ),+ );+ expect(response.body).toEqual({+ ...REFUSAL,+ agent_hints: ["Out of credits."],+ });+ expect(ms).toBeLessThan(LONG / 2);+ });+});diff --git a/apps/api/src/middlewares/agent-hints.ts b/apps/api/src/middlewares/agent-hints.tsindex 35b5d810c..7a93b321b 100644--- a/apps/api/src/middlewares/agent-hints.ts+++ b/apps/api/src/middlewares/agent-hints.ts@@ -55,6 +55,24 @@ function agentHintsFor(return hints;}+/**+ * How long an insufficient-credits refusal may wait for the in-flight provider+ * lookup: whatever is left of the provider timeout since the lookup started.+ * The refusal is sent a few milliseconds after auth, so without this wait it+ * would never carry rule hints on a cold cache. Every other response is sent+ * as soon as the handler calls res.json.+ */+function providerWaitMs(res: Response): number {+ const provider: ProviderHintsHolder | undefined =+ res.locals.agentHintsProvider;+ if (res.locals.insufficientCredits !== true || !provider || provider.settled)+ return 0;+ return Math.min(+ config.AGENT_HINTS_PROVIDER_TIMEOUT_MS,+ provider.startedAt + config.AGENT_HINTS_PROVIDER_TIMEOUT_MS - Date.now(),+ );+}+/*** Only registered on business POST routes, never feedback or polling routes.* Hint computation must never change the response otherwise: any error sends@@ -66,9 +84,7 @@ export function agentHintsMiddleware(return (req, res, next) => {if (!agentHintsRequested(req)) return next();const json = res.json;- res.json = function (body) {- if (!body || typeof body !== "object" || Array.isArray(body))- return json.call(this, body);+ const send = (body: Record<string, unknown>) => {let output = body;try {const hints = agentHintsFor(req, res, endpoint, body);@@ -81,7 +97,33 @@ export function agentHintsMiddleware(error: error instanceof Error ? error.message : String(error),});}- return json.call(this, output);+ return json.call(res, output);+ };+ res.json = function (body) {+ if (!body || typeof body !== "object" || Array.isArray(body))+ return json.call(this, body);+ const waitMs = providerWaitMs(res);+ if (waitMs <= 0) return send(body);+ // Deferred send: the only caller of this path returns right after+ // res.json and never touches the response again, so returning before+ // the body is written is safe. The wait is bounded and never rejects.+ let timer: ReturnType<typeof setTimeout> | undefined;+ const deadline = new Promise<void>(resolve => {+ timer = setTimeout(resolve, waitMs);+ });+ void Promise.race([res.locals.agentHintsProvider.ready, deadline])+ .then(() => {+ clearTimeout(timer);+ if (!res.writableEnded) send(body);+ })+ .catch(error => {+ logger.warn("Deferred response could not be sent", {+ module: "agent-hints",+ endpoint,+ error: error instanceof Error ? error.message : String(error),+ });+ });+ return this;};next();};diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 2c153622b..4afdc771a 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -261,6 +261,7 @@ export function checkCreditsMiddleware(!res.headersSent &&req.auth.team_id !== "8c528896-7882-4587-a4b6-768b721b0b53") {+ res.locals.insufficientCredits = true;return res.status(402).json({success: false,error: