Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): let insufficient-credits responses wait briefly for in-flight hint rules (#5094)

Rakshith Ramprakash5241178e9444
diff --git a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts b/apps/api/src/__tests__/routes/check-credits-middleware.test.ts
index 54f79dbf7..724fc8746 100644
--- a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts
+++ b/apps/api/src/__tests__/routes/check-credits-middleware.test.ts
@@ -108,6 +108,7 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {
expect(res.status).not.toHaveBeenCalled();
expect(req.account.remainingCredits).toBe(Infinity);
expect(res.locals.agentCreditsRemaining).toBe(0);
+ expect(res.locals.insufficientCredits).toBeUndefined();
// request body limit must NOT have been clamped down to 0
expect(req.body.limit).toBe(100);
});
@@ -120,6 +121,7 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {
expect(res.status).toHaveBeenCalledWith(402);
expect(res.locals.agentCreditsRemaining).toBe(0);
+ expect(res.locals.insufficientCredits).toBe(true);
expect(logger.error).toHaveBeenCalledWith(
expect.any(String),
expect.not.objectContaining({ request: expect.anything() }),
diff --git a/apps/api/src/lib/agent-hints-provider.test.ts b/apps/api/src/lib/agent-hints-provider.test.ts
index f2f0e7553..c2ad7d7f4 100644
--- a/apps/api/src/lib/agent-hints-provider.test.ts
+++ b/apps/api/src/lib/agent-hints-provider.test.ts
@@ -110,11 +110,29 @@ describe("external agent hints provider", () => {
it("returns an unsettled holder immediately and fills it once the provider answers", async () => {
const holder = getProviderHints(context());
- expect(holder).toEqual({ settled: false, hints: [], rules: [] });
+ expect(holder).toEqual({
+ settled: false,
+ hints: [],
+ rules: [],
+ ready: expect.any(Promise),
+ startedAt: expect.any(Number),
+ });
await settle(holder);
expect(holder!.hints).toEqual([{ id: "h1", text: "Provider hint." }]);
});
+ it("resolves ready only once the holder is settled, also on failure", async () => {
+ const holder = getProviderHints(context());
+ await holder!.ready;
+ expect(holder!.settled).toBe(true);
+ expect(holder!.hints).toEqual([{ id: "h1", text: "Provider hint." }]);
+
+ handler = reply(500, "nope");
+ const failed = getProviderHints(context());
+ await failed!.ready;
+ expect(failed).toMatchObject({ settled: true, hints: [], rules: [] });
+ });
+
it("sends exactly the contract fields and never the API key", async () => {
config.AGENT_HINTS_PROVIDER_SECRET = "shared-secret";
await settle(
diff --git a/apps/api/src/lib/agent-hints-provider.ts b/apps/api/src/lib/agent-hints-provider.ts
index c1a0d05de..a151a71f2 100644
--- a/apps/api/src/lib/agent-hints-provider.ts
+++ b/apps/api/src/lib/agent-hints-provider.ts
@@ -11,9 +11,11 @@ import { logger } from "./logger";
* Optional external agent hints provider. When AGENT_HINTS_PROVIDER_URL is
* set, hint-enabled requests ask a separate HTTP service for additional
* guidance: finished hints and response guidance rules. The lookup starts
- * after auth and is never awaited: the response only uses provider data that
- * has already arrived (or is cached) by the time it is sent. Every failure
- * means "no provider hints and no rules".
+ * after auth and is not awaited, with one exception: an insufficient-credits
+ * refusal waits for an in-flight lookup, bounded by the provider timeout,
+ * because it is sent before the lookup could otherwise return. Every other
+ * response only uses provider data that has already arrived (or is cached) by
+ * the time it is sent. Every failure means "no provider hints and no rules".
*/
export type ProviderHint = { id: string; text: string };
@@ -22,6 +24,10 @@ export type ProviderHintsHolder = {
settled: boolean;
hints: ProviderHint[];
rules: AgentHintRule[];
+ /** Resolves once settled; never rejects. */
+ ready: Promise<void>;
+ /** When the lookup started (Date.now()); it settles within the provider timeout of this. */
+ startedAt: number;
};
export type AgentHintsProviderContext = {
@@ -207,18 +213,24 @@ export function getProviderHints(
const oldest = cache.keys().next().value;
if (oldest !== undefined) cache.delete(oldest);
}
- const entry: CacheEntry = {
- holder: { settled: false, hints: [], rules: [] },
- expiresAt: Infinity,
+ const holder: ProviderHintsHolder = {
+ settled: false,
+ hints: [],
+ rules: [],
+ ready: Promise.resolve(),
+ startedAt: now,
};
+ const entry: CacheEntry = { holder, expiresAt: Infinity };
cache.set(key, entry);
- void fetchProviderHints(url, context).then(({ hints, rules, ttlMs }) => {
- entry.holder.hints = hints;
- entry.holder.rules = rules;
- entry.holder.settled = true;
- entry.expiresAt = Date.now() + ttlMs;
- });
- return entry.holder;
+ holder.ready = fetchProviderHints(url, context).then(
+ ({ hints, rules, ttlMs }) => {
+ holder.hints = hints;
+ holder.rules = rules;
+ holder.settled = true;
+ entry.expiresAt = Date.now() + ttlMs;
+ },
+ );
+ return holder;
}
/**
diff --git a/apps/api/src/middlewares/__tests__/agent-hints.test.ts b/apps/api/src/middlewares/__tests__/agent-hints.test.ts
index 23299a8b9..3a2196ac3 100644
--- a/apps/api/src/middlewares/__tests__/agent-hints.test.ts
+++ b/apps/api/src/middlewares/__tests__/agent-hints.test.ts
@@ -1,4 +1,5 @@
import express from "express";
+import { performance } from "node:perf_hooks";
import request from "supertest";
import { config } from "../../config";
import type { AgentHintRule } from "../../lib/agent-hint-rules";
@@ -337,7 +338,22 @@ const EXCERPT_BODY = {
data: { web: [{ url: "https://example.com", description: "excerpt" }] },
};
-function holderAppFor(holder: ProviderHintsHolder | undefined, body: object) {
+type TestHolder = Omit<ProviderHintsHolder, "ready" | "startedAt"> &
+ Partial<Pick<ProviderHintsHolder, "ready" | "startedAt">>;
+
+function holderAppFor(
+ holder: TestHolder | undefined,
+ body: object,
+ {
+ status = 200,
+ insufficientCredits = false,
+ remainingCredits,
+ }: {
+ status?: number;
+ insufficientCredits?: boolean;
+ remainingCredits?: number;
+ } = {},
+) {
const app = express();
app.use(express.json());
app.post(
@@ -345,11 +361,17 @@ function holderAppFor(holder: ProviderHintsHolder | undefined, body: object) {
agentHintsMiddleware("search"),
(req, res, next) => {
(req as any).auth = { team_id: "account-team" };
+ if (holder) {
+ holder.ready ??= Promise.resolve();
+ holder.startedAt ??= Date.now();
+ }
res.locals.agentHintsProvider = holder;
+ if (insufficientCredits) res.locals.insufficientCredits = true;
+ res.locals.agentCreditsRemaining = remainingCredits;
next();
},
(_req, res) => {
- res.json(body);
+ res.status(status).json(body);
},
);
return app;
@@ -721,3 +743,146 @@ describe("agent hints provider middleware", () => {
expect(fetchSpy).toHaveBeenCalledTimes(1);
});
});
+
+describe("insufficient-credits refusals and a pending provider lookup", () => {
+ // Timing assertions use a wide margin between "waited for the bound" and
+ // "did not wait": a long bound (LONG) that a correct response never reaches,
+ // or a short one (SHORT) checked only from below, so scheduling delay under
+ // load cannot fail a correct response.
+ const LONG = 10_000;
+ const SHORT = 60;
+ const REFUSAL = { success: false, error: "Insufficient credits." };
+ const CREDIT_RULES: AgentHintRule[] = [
+ {
+ id: "out",
+ group: "credits",
+ when: [
+ { signal: "success", op: "eq", value: false },
+ { signal: "remaining_credits", op: "lt", value: 1 },
+ ],
+ text: "Out of credits.",
+ },
+ {
+ id: "low",
+ group: "credits",
+ when: [{ signal: "remaining_credits", op: "lt", value: 100 }],
+ text: "Low on credits.",
+ },
+ ];
+ const originalTimeout = config.AGENT_HINTS_PROVIDER_TIMEOUT_MS;
+
+ beforeEach(() => {
+ config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = LONG;
+ });
+
+ afterAll(() => {
+ config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = originalTimeout;
+ });
+
+ /** A lookup that settles with CREDIT_RULES after `ms`, or never. */
+ function pendingHolder(ms?: number): TestHolder {
+ const holder: TestHolder = {
+ settled: false,
+ hints: [],
+ rules: [],
+ startedAt: Date.now(),
+ };
+ holder.ready = new Promise<void>(resolve => {
+ if (ms === undefined) return;
+ setTimeout(() => {
+ holder.rules = CREDIT_RULES;
+ holder.settled = true;
+ resolve();
+ }, ms);
+ });
+ return holder;
+ }
+
+ async function timed(app: express.Express) {
+ const started = performance.now();
+ const response = await request(app)
+ .post("/")
+ .set("X-Firecrawl-Agent-Hints", "true")
+ .send({});
+ return { response, ms: performance.now() - started };
+ }
+
+ const REFUSED = {
+ status: 402,
+ insufficientCredits: true,
+ remainingCredits: 0,
+ };
+
+ it("waits for a lookup that settles within the bound and adds its hints", async () => {
+ const { response, ms } = await timed(
+ holderAppFor(pendingHolder(15), REFUSAL, REFUSED),
+ );
+ expect(response.statusCode).toBe(402);
+ expect(response.body).toEqual({
+ ...REFUSAL,
+ agent_hints: ["Out of credits."],
+ });
+ expect(ms).toBeLessThan(LONG / 2);
+ });
+
+ it("sends the refusal at the bound without hints when the lookup is slower", async () => {
+ config.AGENT_HINTS_PROVIDER_TIMEOUT_MS = SHORT;
+ const { response, ms } = await timed(
+ holderAppFor(pendingHolder(), REFUSAL, REFUSED),
+ );
+ expect(response.statusCode).toBe(402);
+ expect(response.body).toEqual(REFUSAL);
+ expect(ms).toBeGreaterThanOrEqual(SHORT - 5);
+ });
+
+ it("counts the bound from when the lookup started", async () => {
+ const holder = pendingHolder();
+ holder.startedAt = Date.now() - (LONG - 20);
+ const { response, ms } = await timed(
+ holderAppFor(holder, REFUSAL, REFUSED),
+ );
+ expect(response.body).toEqual(REFUSAL);
+ expect(ms).toBeLessThan(LONG / 2);
+ });
+
+ it("does not delay a successful response while the lookup is pending", async () => {
+ const body = { success: true, data: {} };
+ const { response, ms } = await timed(
+ holderAppFor(pendingHolder(), body, { remainingCredits: 5 }),
+ );
+ expect(response.statusCode).toBe(200);
+ expect(response.body).toEqual(body);
+ expect(ms).toBeLessThan(LONG / 2);
+ });
+
+ it("does not delay other refusals while the lookup is pending", async () => {
+ const { response, ms } = await timed(
+ holderAppFor(pendingHolder(), REFUSAL, {
+ status: 402,
+ remainingCredits: 0,
+ }),
+ );
+ expect(response.body).toEqual(REFUSAL);
+ expect(ms).toBeLessThan(LONG / 2);
+ });
+
+ it("does not wait when the lookup has already settled", async () => {
+ const { response, ms } = await timed(
+ holderAppFor(
+ {
+ settled: true,
+ hints: [],
+ rules: CREDIT_RULES,
+ ready: new Promise(() => {}),
+ },
+ REFUSAL,
+ REFUSED,
+ ),
+ );
+ expect(response.body).toEqual({
+ ...REFUSAL,
+ agent_hints: ["Out of credits."],
+ });
+ expect(ms).toBeLessThan(LONG / 2);
+ });
+});
diff --git a/apps/api/src/middlewares/agent-hints.ts b/apps/api/src/middlewares/agent-hints.ts
index 35b5d810c..7a93b321b 100644
--- a/apps/api/src/middlewares/agent-hints.ts
+++ b/apps/api/src/middlewares/agent-hints.ts
@@ -55,6 +55,24 @@ function agentHintsFor(
return hints;
}
+/**
+ * How long an insufficient-credits refusal may wait for the in-flight provider
+ * lookup: whatever is left of the provider timeout since the lookup started.
+ * The refusal is sent a few milliseconds after auth, so without this wait it
+ * would never carry rule hints on a cold cache. Every other response is sent
+ * as soon as the handler calls res.json.
+ */
+function providerWaitMs(res: Response): number {
+ const provider: ProviderHintsHolder | undefined =
+ res.locals.agentHintsProvider;
+ if (res.locals.insufficientCredits !== true || !provider || provider.settled)
+ return 0;
+ return Math.min(
+ config.AGENT_HINTS_PROVIDER_TIMEOUT_MS,
+ provider.startedAt + config.AGENT_HINTS_PROVIDER_TIMEOUT_MS - Date.now(),
+ );
+}
+
/**
* Only registered on business POST routes, never feedback or polling routes.
* Hint computation must never change the response otherwise: any error sends
@@ -66,9 +84,7 @@ export function agentHintsMiddleware(
return (req, res, next) => {
if (!agentHintsRequested(req)) return next();
const json = res.json;
- res.json = function (body) {
- if (!body || typeof body !== "object" || Array.isArray(body))
- return json.call(this, body);
+ const send = (body: Record<string, unknown>) => {
let output = body;
try {
const hints = agentHintsFor(req, res, endpoint, body);
@@ -81,7 +97,33 @@ export function agentHintsMiddleware(
error: error instanceof Error ? error.message : String(error),
});
}
- return json.call(this, output);
+ return json.call(res, output);
+ };
+ res.json = function (body) {
+ if (!body || typeof body !== "object" || Array.isArray(body))
+ return json.call(this, body);
+ const waitMs = providerWaitMs(res);
+ if (waitMs <= 0) return send(body);
+ // Deferred send: the only caller of this path returns right after
+ // res.json and never touches the response again, so returning before
+ // the body is written is safe. The wait is bounded and never rejects.
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const deadline = new Promise<void>(resolve => {
+ timer = setTimeout(resolve, waitMs);
+ });
+ void Promise.race([res.locals.agentHintsProvider.ready, deadline])
+ .then(() => {
+ clearTimeout(timer);
+ if (!res.writableEnded) send(body);
+ })
+ .catch(error => {
+ logger.warn("Deferred response could not be sent", {
+ module: "agent-hints",
+ endpoint,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ });
+ return this;
};
next();
};
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index 2c153622b..4afdc771a 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -261,6 +261,7 @@ export function checkCreditsMiddleware(
!res.headersSent &&
req.auth.team_id !== "8c528896-7882-4587-a4b6-768b721b0b53"
) {
+ res.locals.insufficientCredits = true;
return res.status(402).json({
success: false,
error: