Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): apply per-operation research timeouts
Karan Lokchandani9f6d7075a00d
diff --git a/apps/api/src/controllers/v2/__tests__/research-timeouts.test.ts b/apps/api/src/controllers/v2/__tests__/research-timeouts.test.tsnew file mode 100644index 000000000..6a1139c3f--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/research-timeouts.test.ts@@ -0,0 +1,180 @@+import { vi } from "vitest";++const mocks = vi.hoisted(() => ({+ logRequest: vi.fn(),+ logResearchEndpoint: vi.fn(),+ fetchResearchUpstream: vi.fn(),+ chargeKeylessCredits: vi.fn(),+}));++vi.mock("../../../services/logging/log_job", () => ({+ logRequest: mocks.logRequest,+ logResearchEndpoint: mocks.logResearchEndpoint,+}));++vi.mock("../../../lib/research-upstream", () => ({+ fetchResearchUpstream: mocks.fetchResearchUpstream,+}));++vi.mock("../../../services/billing/credit_billing", () => ({+ billTeam: vi.fn(),+}));++vi.mock("../../../lib/keyless", () => ({+ chargeKeylessCredits: mocks.chargeKeylessCredits,+}));++vi.mock("../../../lib/logger", () => ({+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ child: vi.fn().mockReturnThis(),+ },+}));++import { createDeveloperRouter, createResearchRouter } from "../research-proxy";++const TEAM_ID = "11111111-1111-1111-1111-111111111111";++const flush = () => new Promise(resolve => setImmediate(resolve));++function routeHandler(router: any, path: string) {+ const layer = router.stack.find(+ (candidate: any) =>+ candidate.route?.path === path && candidate.route?.methods?.get,+ );+ return layer.route.stack[0].handle;+}++function makeReq(query: Record<string, unknown>, params = {}) {+ return {+ method: "GET",+ query,+ params,+ body: {},+ headers: {},+ auth: { team_id: TEAM_ID },+ acuc: { api_key_id: 7, flags: {} },+ } as any;+}++function makeRes() {+ const res: any = {+ status: vi.fn(),+ json: vi.fn(),+ send: vi.fn(),+ end: vi.fn(),+ setHeader: vi.fn(),+ };+ res.status.mockReturnValue(res);+ res.json.mockReturnValue(res);+ res.send.mockReturnValue(res);+ return res;+}++async function callRoute(+ router: any,+ path: string,+ query: Record<string, unknown>,+ params = {},+) {+ const res = makeRes();+ routeHandler(router, path)(makeReq(query, params), res, vi.fn());+ await flush();+ return res;+}++beforeEach(() => {+ vi.clearAllMocks();+ mocks.logRequest.mockResolvedValue(undefined);+ mocks.logResearchEndpoint.mockResolvedValue(undefined);+ mocks.chargeKeylessCredits.mockResolvedValue(undefined);+ mocks.fetchResearchUpstream.mockResolvedValue({+ ok: true,+ status: 200,+ headers: new Headers(),+ text: async () => JSON.stringify({ success: true, results: [] }),+ });+});++describe("research upstream timeouts", () => {+ it.each([+ {+ name: "paper search",+ router: () => createResearchRouter(),+ path: "/papers",+ query: { query: "graph databases" },+ params: {},+ timeoutMs: 30_000,+ },+ {+ name: "similar papers",+ router: () => createResearchRouter(),+ path: "/papers/:id/similar",+ query: { intent: "related work" },+ params: { id: "paper-1" },+ timeoutMs: 10_000,+ },+ {+ name: "GitHub search",+ router: () => createResearchRouter(),+ path: "/github",+ query: { query: "retry logic" },+ params: {},+ timeoutMs: 12_000,+ },+ {+ name: "developer search",+ router: () => createDeveloperRouter(),+ path: "/search",+ query: { query: "http client" },+ params: {},+ timeoutMs: 15_000,+ },+ ])("applies the $name timeout", async testCase => {+ await callRoute(+ testCase.router(),+ testCase.path,+ testCase.query,+ testCase.params,+ );++ expect(mocks.fetchResearchUpstream).toHaveBeenCalledWith(+ expect.objectContaining({ timeoutMs: testCase.timeoutMs }),+ );+ });++ it.each([+ { name: "inspect", query: {}, timeoutMs: 5_000 },+ { name: "read", query: { query: "summarize methods" }, timeoutMs: 120_000 },+ ])("uses the paper $name timeout", async ({ query, timeoutMs }) => {+ await callRoute(createResearchRouter(), "/papers/:id", query, {+ id: "paper-1",+ });++ expect(mocks.fetchResearchUpstream).toHaveBeenCalledWith(+ expect.objectContaining({ timeoutMs }),+ );+ });++ it("maps an operation timeout to 504", async () => {+ mocks.fetchResearchUpstream.mockRejectedValue(+ new DOMException(+ "The operation was aborted due to timeout",+ "TimeoutError",+ ),+ );++ const res = await callRoute(createResearchRouter(), "/github", {+ query: "retry logic",+ });++ expect(mocks.fetchResearchUpstream).toHaveBeenCalledWith(+ expect.objectContaining({ timeoutMs: 12_000 }),+ );+ expect(res.status).toHaveBeenCalledWith(504);+ expect(res.end).toHaveBeenCalled();+ });+});diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 5a65ae7de..1c3102200 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -23,6 +23,13 @@ import { requestOrigin } from "../../lib/request-origin";const SEARCH_CREDITS_PER_TEN_RESULTS = 2;const ZDR_SEARCH_CREDITS_PER_TEN_RESULTS = 10;+const DEVELOPER_SEARCH_TIMEOUT_MS = 15_000;+const PAPER_SEARCH_TIMEOUT_MS = 30_000;+const PAPER_INSPECT_TIMEOUT_MS = 5_000;+const SIMILAR_PAPERS_TIMEOUT_MS = 10_000;+const GITHUB_SEARCH_TIMEOUT_MS = 12_000;+const PAPER_READ_TIMEOUT_MS = 120_000;+const FORWARDED_REQUEST_HEADERS = ["accept", "x-request-id"];const FORWARDED_RESPONSE_HEADERS = ["content-type", "x-request-id"];@@ -142,6 +149,7 @@ type ResearchEndpointConfig = {params: Record<string, any>,req: RequestWithAuth<any, any, any>,) => string;+ timeoutMs?: number;billAs: "scrape" | "search";};@@ -240,6 +248,7 @@ async function fetchForRequest(path: string,params: Record<string, unknown>,queryKeys: string[],+ timeoutMs?: number,) {const headers: Record<string, string> = {};for (const h of FORWARDED_REQUEST_HEADERS) {@@ -248,7 +257,13 @@ async function fetchForRequest(}headers["firecrawl-team-id"] = req.auth.team_id;- return fetchResearchUpstream({ path, params, queryKeys, headers });+ return fetchResearchUpstream({+ path,+ params,+ queryKeys,+ headers,+ timeoutMs,+ });}function createResearchController(@@ -311,6 +326,7 @@ function createResearchController(endpoint.upstreamPath(params, authedReq),params,queryKeys,+ endpoint.timeoutMs,);if (!upstream) {statusCode = 404;@@ -438,6 +454,7 @@ export function createResearchRouter(options: { legacy?: boolean } = {}) {action: "searchPapers",targetHint: params => String(params.query),upstreamPath: () => "/v2/research/papers",+ timeoutMs: PAPER_SEARCH_TIMEOUT_MS,billAs: "search",},options,@@ -459,6 +476,7 @@ export function createResearchRouter(options: { legacy?: boolean } = {}) {`${req.params.id}: ${String(params.intent)}`,upstreamPath: (_params, req) =>`/v2/research/papers/${encodeURIComponent(req.params.id)}/similar`,+ timeoutMs: SIMILAR_PAPERS_TIMEOUT_MS,billAs: "search",},options,@@ -482,6 +500,7 @@ export function createResearchRouter(options: { legacy?: boolean } = {}) {targetHint: (_params, request) => request.params.id,upstreamPath: (_params, request) =>`/v2/research/papers/${encodeURIComponent(request.params.id)}`,+ timeoutMs: isRead ? PAPER_READ_TIMEOUT_MS : PAPER_INSPECT_TIMEOUT_MS,billAs: "scrape",},options,@@ -502,6 +521,7 @@ export function createResearchRouter(options: { legacy?: boolean } = {}) {action: "searchGithub",targetHint: params => String(params.query),upstreamPath: () => "/v2/research/github",+ timeoutMs: GITHUB_SEARCH_TIMEOUT_MS,billAs: "search",},options,@@ -525,6 +545,7 @@ export function createDeveloperRouter(options: { root?: boolean } = {}) {action: "searchDeveloper",targetHint: params => String(params.query),upstreamPath: () => "/v2/code/search",+ timeoutMs: DEVELOPER_SEARCH_TIMEOUT_MS,billAs: "search",},),diff --git a/apps/api/src/lib/research-upstream.test.ts b/apps/api/src/lib/research-upstream.test.tsnew file mode 100644index 000000000..40b58b437--- /dev/null+++ b/apps/api/src/lib/research-upstream.test.ts@@ -0,0 +1,50 @@+import { vi } from "vitest";++const mocks = vi.hoisted(() => ({+ agent: vi.fn(),+ fetch: vi.fn(),+}));++vi.mock("undici", () => ({+ Agent: mocks.agent,+ fetch: mocks.fetch,+}));++vi.mock("../config", () => ({+ config: { RESEARCH_PROXY_URL: "https://research.test/" },+}));++import { fetchResearchUpstream } from "./research-upstream";++beforeEach(() => {+ mocks.fetch.mockReset();+ mocks.fetch.mockResolvedValue({ ok: true, status: 200 });+});++afterEach(() => {+ vi.restoreAllMocks();+});++describe("fetchResearchUpstream", () => {+ it("keeps the 120 second fallback when no operation timeout is supplied", async () => {+ const signal = new AbortController().signal;+ const timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(signal);++ await fetchResearchUpstream({+ path: "/v2/research/unlisted",+ params: {},+ queryKeys: [],+ headers: {},+ });++ expect(timeout).toHaveBeenCalledWith(120_000);+ expect(mocks.fetch).toHaveBeenCalledWith(+ new URL("https://research.test/v2/research/unlisted"),+ expect.objectContaining({ signal }),+ );+ });++ it("limits connection setup without dispatcher headers or body timeouts", () => {+ expect(mocks.agent).toHaveBeenCalledWith({ connectTimeout: 10_000 });+ });+});diff --git a/apps/api/src/lib/research-upstream.ts b/apps/api/src/lib/research-upstream.tsindex 85a398735..a3f01eade 100644--- a/apps/api/src/lib/research-upstream.ts+++ b/apps/api/src/lib/research-upstream.ts@@ -2,11 +2,10 @@ import { Agent, fetch } from "undici";import { config } from "../config";const TIMEOUT_MS = 120_000;+const CONNECT_TIMEOUT_MS = 10_000;const dispatcher = new Agent({- connectTimeout: TIMEOUT_MS,- headersTimeout: TIMEOUT_MS,- bodyTimeout: TIMEOUT_MS,+ connectTimeout: CONNECT_TIMEOUT_MS,});function appendQuery(