Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(js-sdk): explicit axios timeout (#3440)

tomsideguide0c24965c73d7
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 55ee202d1..f53375e13 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.19.0",
+ "version": "4.20.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape-browser.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape-browser.unit.test.ts
index 1ddc6a747..edf6ad832 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape-browser.unit.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape-browser.unit.test.ts
@@ -14,12 +14,15 @@ describe("JS SDK v2 scrape-browser methods", () => {
}));
const http = { post } as any;
- const response = await interact(http, "job-123", { code: "console.log('ok')" });
-
- expect(post).toHaveBeenCalledWith("/v2/scrape/job-123/interact", {
+ const response = await interact(http, "job-123", {
code: "console.log('ok')",
- language: "node",
});
+
+ expect(post).toHaveBeenCalledWith(
+ "/v2/scrape/job-123/interact",
+ { code: "console.log('ok')", language: "node" },
+ {},
+ );
expect(response.success).toBe(true);
expect(response.exitCode).toBe(0);
});
@@ -38,22 +41,27 @@ describe("JS SDK v2 scrape-browser methods", () => {
}));
const http = { post } as any;
- const response = await interact(http, "job-456", { prompt: "Click the login button" });
-
- expect(post).toHaveBeenCalledWith("/v2/scrape/job-456/interact", {
+ const response = await interact(http, "job-456", {
prompt: "Click the login button",
- language: "node",
});
+
+ expect(post).toHaveBeenCalledWith(
+ "/v2/scrape/job-456/interact",
+ { prompt: "Click the login button", language: "node" },
+ {},
+ );
expect(response.success).toBe(true);
expect(response.output).toBe("Clicked the button");
expect(response.liveViewUrl).toBe("https://live.example.com/view");
- expect(response.interactiveLiveViewUrl).toBe("https://live.example.com/interactive");
+ expect(response.interactiveLiveViewUrl).toBe(
+ "https://live.example.com/interactive",
+ );
});
test("interact throws when neither code nor prompt provided", async () => {
const http = { post: jest.fn() } as any;
await expect(interact(http, "job-123", {})).rejects.toThrow(
- "Either 'code' or 'prompt' must be provided"
+ "Either 'code' or 'prompt' must be provided",
);
});
@@ -68,7 +76,7 @@ describe("JS SDK v2 scrape-browser methods", () => {
const http = { post } as any;
await expect(
- interact(http, "bad-id", { code: "console.log('ok')" })
+ interact(http, "bad-id", { code: "console.log('ok')" }),
).rejects.toBeInstanceOf(SdkError);
});
@@ -98,7 +106,26 @@ describe("JS SDK v2 scrape-browser methods", () => {
const http = { delete: del } as any;
await expect(stopInteraction(http, "job-123")).rejects.toBeInstanceOf(
- SdkError
+ SdkError,
+ );
+ });
+
+ test("interact converts seconds-based body timeout to ms axios timeout", async () => {
+ const post = jest.fn(async () => ({
+ status: 200,
+ data: { success: true, stdout: "ok", exitCode: 0 },
+ }));
+
+ const http = { post } as any;
+ await interact(http, "job-123", {
+ code: "console.log('ok')",
+ timeout: 150,
+ });
+
+ expect(post).toHaveBeenCalledWith(
+ "/v2/scrape/job-123/interact",
+ { code: "console.log('ok')", language: "node", timeout: 150 },
+ { timeoutMs: 155000 },
);
});
});
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
index d21f23294..90bfd4665 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
@@ -11,7 +11,11 @@ import {
import { HttpClient } from "../utils/httpClient";
import { ensureValidScrapeOptions } from "../utils/validation";
import { fetchAllPages } from "../utils/pagination";
-import { normalizeAxiosError, throwForBadResponse, isRetryableError } from "../utils/errorHandler";
+import {
+ normalizeAxiosError,
+ throwForBadResponse,
+ isRetryableError,
+} from "../utils/errorHandler";
export async function startBatchScrape(
http: HttpClient,
@@ -26,9 +30,10 @@ export async function startBatchScrape(
idempotencyKey,
integration,
origin,
- }: BatchScrapeOptions = {}
+ }: BatchScrapeOptions = {},
): Promise<BatchScrapeResponse> {
- if (!Array.isArray(urls) || urls.length === 0) throw new Error("URLs list cannot be empty");
+ if (!Array.isArray(urls) || urls.length === 0)
+ throw new Error("URLs list cannot be empty");
const payload: Record<string, unknown> = { urls };
if (options) {
ensureValidScrapeOptions(options);
@@ -39,16 +44,29 @@ export async function startBatchScrape(
if (ignoreInvalidURLs != null) payload.ignoreInvalidURLs = ignoreInvalidURLs;
if (maxConcurrency != null) payload.maxConcurrency = maxConcurrency;
if (zeroDataRetention != null) payload.zeroDataRetention = zeroDataRetention;
- if (integration != null && integration.trim()) payload.integration = integration.trim();
+ if (integration != null && integration.trim())
+ payload.integration = integration.trim();
if (origin) payload.origin = origin;
try {
const headers = http.prepareHeaders(idempotencyKey);
- const res = await http.post<{ success: boolean; id: string; url: string; invalidURLs?: string[]; error?: string }>("/v2/batch/scrape", payload, headers);
- if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "start batch scrape");
- return { id: res.data.id, url: res.data.url, invalidURLs: res.data.invalidURLs || undefined };
+ const res = await http.post<{
+ success: boolean;
+ id: string;
+ url: string;
+ invalidURLs?: string[];
+ error?: string;
+ }>("/v2/batch/scrape", payload, { headers });
+ if (res.status !== 200 || !res.data?.success)
+ throwForBadResponse(res, "start batch scrape");
+ return {
+ id: res.data.id,
+ url: res.data.url,
+ invalidURLs: res.data.invalidURLs || undefined,
+ };
} catch (err: any) {
- if (err?.isAxiosError) return normalizeAxiosError(err, "start batch scrape");
+ if (err?.isAxiosError)
+ return normalizeAxiosError(err, "start batch scrape");
throw err;
}
}
@@ -56,11 +74,21 @@ export async function startBatchScrape(
export async function getBatchScrapeStatus(
http: HttpClient,
jobId: string,
- pagination?: PaginationConfig
+ pagination?: PaginationConfig,
): Promise<BatchScrapeJob> {
try {
- const res = await http.get<{ success: boolean; status: BatchScrapeJob["status"]; completed?: number; total?: number; creditsUsed?: number; expiresAt?: string; next?: string | null; data?: Document[] }>(`/v2/batch/scrape/${jobId}`);
- if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get batch scrape status");
+ const res = await http.get<{
+ success: boolean;
+ status: BatchScrapeJob["status"];
+ completed?: number;
+ total?: number;
+ creditsUsed?: number;
+ expiresAt?: string;
+ next?: string | null;
+ data?: Document[];
+ }>(`/v2/batch/scrape/${jobId}`);
+ if (res.status !== 200 || !res.data?.success)
+ throwForBadResponse(res, "get batch scrape status");
const body = res.data;
const initialDocs = (body.data || []) as Document[];
const auto = pagination?.autoPaginate ?? true;
@@ -77,7 +105,12 @@ export async function getBatchScrapeStatus(
};
}
- const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);
+ const aggregated = await fetchAllPages(
+ http,
+ body.next,
+ initialDocs,
+ pagination,
+ );
return {
id: jobId,
status: body.status,
@@ -89,35 +122,57 @@ export async function getBatchScrapeStatus(
data: aggregated,
};
} catch (err: any) {
- if (err?.isAxiosError) return normalizeAxiosError(err, "get batch scrape status");
+ if (err?.isAxiosError)
+ return normalizeAxiosError(err, "get batch scrape status");
throw err;
}
}
-export async function cancelBatchScrape(http: HttpClient, jobId: string): Promise<boolean> {
+export async function cancelBatchScrape(
+ http: HttpClient,
+ jobId: string,
+): Promise<boolean> {
try {
- const res = await http.delete<{ status: string }>(`/v2/batch/scrape/${jobId}`);
+ const res = await http.delete<{ status: string }>(
+ `/v2/batch/scrape/${jobId}`,
+ );
if (res.status !== 200) throwForBadResponse(res, "cancel batch scrape");
return res.data?.status === "cancelled";
} catch (err: any) {
- if (err?.isAxiosError) return normalizeAxiosError(err, "cancel batch scrape");
+ if (err?.isAxiosError)
+ return normalizeAxiosError(err, "cancel batch scrape");
throw err;
}
}
-export async function getBatchScrapeErrors(http: HttpClient, jobId: string): Promise<CrawlErrorsResponse> {
+export async function getBatchScrapeErrors(
+ http: HttpClient,
+ jobId: string,
+): Promise<CrawlErrorsResponse> {
try {
- const res = await http.get<{ success?: boolean; data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] } }>(`/v2/batch/scrape/${jobId}/errors`);
+ const res = await http.get<{
+ success?: boolean;
+ data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] };
+ }>(`/v2/batch/scrape/${jobId}/errors`);
if (res.status !== 200) throwForBadResponse(res, "get batch scrape errors");
const payload = res.data?.data ?? (res.data as any);
- return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };
+ return {
+ errors: payload.errors || [],
+ robotsBlocked: payload.robotsBlocked || [],
+ };
} catch (err: any) {
- if (err?.isAxiosError) return normalizeAxiosError(err, "get batch scrape errors");
+ if (err?.isAxiosError)
+ return normalizeAxiosError(err, "get batch scrape errors");
throw err;
}
}
-export async function waitForBatchCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<BatchScrapeJob> {
+export async function waitForBatchCompletion(
+ http: HttpClient,
+ jobId: string,
+ pollInterval = 2,
+ timeout?: number,
+): Promise<BatchScrapeJob> {
const start = Date.now();
while (true) {
@@ -137,7 +192,7 @@ export async function waitForBatchCompletion(http: HttpClient, jobId: string, po
err.status,
err.code,
err.details,
- jobId
+ jobId,
);
throw errorWithJobId;
}
@@ -147,24 +202,30 @@ export async function waitForBatchCompletion(http: HttpClient, jobId: string, po
}
if (timeout != null && Date.now() - start > timeout * 1000) {
- throw new JobTimeoutError(jobId, timeout, 'batch');
+ throw new JobTimeoutError(jobId, timeout, "batch");
}
-
- await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
+
+ await new Promise(r => setTimeout(r, Math.max(1000, pollInterval * 1000)));
}
}
export async function batchScrape(
http: HttpClient,
urls: string[],
- opts: BatchScrapeOptions & { pollInterval?: number; timeout?: number } = {}
+ opts: BatchScrapeOptions & { pollInterval?: number; timeout?: number } = {},
): Promise<BatchScrapeJob> {
const start = await startBatchScrape(http, urls, opts);
- return waitForBatchCompletion(http, start.id, opts.pollInterval ?? 2, opts.timeout);
+ return waitForBatchCompletion(
+ http,
+ start.id,
+ opts.pollInterval ?? 2,
+ opts.timeout,
+ );
}
export function chunkUrls(urls: string[], chunkSize = 100): string[][] {
const chunks: string[][] = [];
- for (let i = 0; i < urls.length; i += chunkSize) chunks.push(urls.slice(i, i + chunkSize));
+ for (let i = 0; i < urls.length; i += chunkSize)
+ chunks.push(urls.slice(i, i + chunkSize));
return chunks;
}
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/browser.ts b/apps/js-sdk/firecrawl/src/v2/methods/browser.ts
index 31a39ba13..032f1e99c 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/browser.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/browser.ts
@@ -5,7 +5,10 @@ import type {
BrowserListResponse,
} from "../types";
import { HttpClient } from "../utils/httpClient";
-import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+import {
+ normalizeAxiosError,
+ throwForBadResponse,
+} from "../utils/errorHandler";
export async function browser(
http: HttpClient,
@@ -19,7 +22,7 @@ export async function browser(
};
integration?: string;
origin?: string;
- } = {}
+ } = {},
): Promise<BrowserCreateResponse> {
const body: Record<string, unknown> = {};
if (args.ttl != null) body.ttl = args.ttl;
@@ -34,7 +37,8 @@ export async function browser(
if (res.status !== 200) throwForBadResponse(res, "create browser session");
return res.data;
} catch (err: any) {
- if (err?.isAxiosError) return normalizeAxiosError(err, "create browser session");
+ if (err?.isAxiosError)
+ return normalizeAxiosError(err, "create browser session");
throw err;
}
}
@@ -46,7 +50,7 @@ export async function browserExecute(
code: string;
language?: "python" | "node" | "bash";
timeout?: number;
- }
+ },
): Promise<BrowserExecuteResponse> {
const body: Record<string, unknown> = {
code: args.code,
@@ -57,28 +61,31 @@ export async function browserExecute(
try {
const res = await http.post<BrowserExecuteResponse>(
`/v2/browser/${sessionId}/execute`,
- body
+ body,
+ args.timeout != null ? { timeoutMs: args.timeout * 1000 + 5000 } : {},
);
if (res.status !== 200) throwForBadResponse(res, "execute browser code");
return res.data;
} catch (err: any) {
- if (err?.isAxiosError) return normalizeAxiosError(err, "execute browser code");
+ if (err?.isAxiosError)
+ return normalizeAxiosError(err, "execute browser code");
throw err;
}
}
export async function deleteBrowser(
http: HttpClient,
- sessionId: string
+ sessionId: string,
): Promise<BrowserDeleteResponse> {
try {
const res = await http.delete<BrowserDeleteResponse>(
- `/v2/browser/${sessionId}`
+ `/v2/browser/${sessionId}`,
);
if (res.status !== 200) throwForBadResponse(res, "delete browser session");
return res.data;
} catch (err: any) {
- if (err?.isAxiosError) return normalizeAxiosError(err, "delete browser session");
+ if (err?.isAxiosError)
+ return normalizeAxiosError(err, "delete browser session");
throw err;
}
}
@@ -87,7 +94,7 @@ export async function listBrowsers(
http: HttpClient,
args: {
status?: "active" | "destroyed";
- } = {}
+ } = {},
): Promise<BrowserListResponse> {
let endpoint = "/v2/browser";
if (args.status) endpoint += `?status=${args.status}`;
@@ -97,7 +104,8 @@ export async function listBrowsers(
if (res.status !== 200) throwForBadResponse(res, "list browser sessions");
return res.data;
} catch (err: any) {
- if (err?.isAxiosError) return normalizeAxiosError(err, "list browser sessions");
+ if (err?.isAxiosError)
+ return normalizeAxiosError(err, "list browser sessions");
throw err;
}
}
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/map.ts b/apps/js-sdk/firecrawl/src/v2/methods/map.ts
index 18d0b8938..7958fb361 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/map.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/map.ts
@@ -1,28 +1,51 @@
import { type MapData, type MapOptions, type SearchResultWeb } from "../types";
import { HttpClient } from "../utils/httpClient";
-import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";
+import {
+ throwForBadResponse,
+ normalizeAxiosError,
+} from "../utils/errorHandler";
-function prepareMapPayload(url: string, options?: MapOptions): Record<string, unknown> {
+function prepareMapPayload(
+ url: string,
+ options?: MapOptions,
+): Record<string, unknown> {
if (!url || !url.trim()) throw new Error("URL cannot be empty");
const payload: Record<string, unknown> = { url: url.trim() };
if (options) {
if (options.sitemap != null) payload.sitemap = options.sitemap;
if (options.search != null) payload.search = options.search;
- if (options.includeSubdomains != null) payload.includeSubdomains = options.includeSubdomains;
- if (options.ignoreQueryParameters != null) payload.ignoreQueryParameters = options.ignoreQueryParameters;
+ if (options.includeSubdomains != null)
+ payload.includeSubdomains = options.includeSubdomains;
+ if (options.ignoreQueryParameters != null)
+ payload.ignoreQueryParameters = options.ignoreQueryParameters;
if (options.limit != null) payload.limit = options.limit;
if (options.timeout != null) payload.timeout = options.timeout;
- if (options.integration != null && options.integration.trim()) payload.integration = options.integration.trim();
+ if (options.integration != null && options.integration.trim())
+ payload.integration = options.integration.trim();
if (options.origin) payload.origin = options.origin;
if (options.location != null) payload.location = options.location;
}
return payload;
}
-export async function map(http: HttpClient, url: string, options?: MapOptions): Promise<MapData> {
+export async function map(
+ http: HttpClient,
+ url: string,
+ options?: MapOptions,
+): Promise<MapData> {
const payload = prepareMapPayload(url, options);
try {
- const res = await http.post<{ success: boolean; error?: string; links?: Array<string | SearchResultWeb> }>("/v2/map", payload);
+ const res = await http.post<{
+ success: boolean;
+ error?: string;
+ links?: Array<string | SearchResultWeb>;
+ }>(
+ "/v2/map",
+ payload,
+ typeof options?.timeout === "number"
+ ? { timeoutMs: options.timeout + 5000 }
+ : {},
+ );
if (res.status !== 200 || !res.data?.success) {
throwForBadResponse(res, "map");
}
@@ -30,7 +53,12 @@ export async function map(http: HttpClient, url: string, options?: MapOptions):
const links: SearchResultWeb[] = [];
for (const item of linksIn) {
if (typeof item === "string") links.push({ url: item });
- else if (item && typeof item === "object") links.push({ url: item.url, title: (item as any).title, description: (item as any).description });
+ else if (item && typeof item === "object")
+ links.push({
+ url: item.url,
+ title: (item as any).title,
+ description: (item as any).description,
+ });
}
return { links };
} catch (err: any) {
@@ -38,4 +66,3 @@ export async function map(http: HttpClient, url: string, options?: MapOptions):
throw err;
}
}
-
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/parse.ts b/apps/js-sdk/firecrawl/src/v2/methods/parse.ts
index b8734d3d2..2de04ab1d 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/parse.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/parse.ts
@@ -1,7 +1,10 @@
import { type Document, type ParseFile, type ParseOptions } from "../types";
import { HttpClient } from "../utils/httpClient";
import { ensureValidParseOptions } from "../utils/validation";
-import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";
+import {
+ throwForBadResponse,
+ normalizeAxiosError,
+} from "../utils/errorHandler";
import { getVersion } from "../utils/getVersion";
function toUploadBlob(input: ParseFile["data"], contentType?: string): Blob {
@@ -25,7 +28,9 @@ function toUploadBlob(input: ParseFile["data"], contentType?: string): Blob {
}
if (typeof input === "string") {
- return new Blob([input], { type: contentType ?? "text/plain; charset=utf-8" });
+ return new Blob([input], {
+ type: contentType ?? "text/plain; charset=utf-8",
+ });
}
throw new Error("Unsupported parse file data type");
@@ -57,7 +62,7 @@ export async function parse(
origin:
typeof options?.origin === "string" && options.origin.includes("mcp")
? options.origin
- : options?.origin ?? `js-sdk@${version}`,
+ : (options?.origin ?? `js-sdk@${version}`),
};
const formData = new FormData();
@@ -68,17 +73,18 @@ export async function parse(
file.filename.trim(),
);
- const requestTimeoutMs =
- typeof normalizedOptions.timeout === "number"
- ? normalizedOptions.timeout + 5000
- : undefined;
-
try {
const res = await http.postMultipart<{
success: boolean;
data?: Document;
error?: string;
- }>("/v2/parse", formData, undefined, requestTimeoutMs);
+ }>(
+ "/v2/parse",
+ formData,
+ typeof normalizedOptions.timeout === "number"
+ ? { timeoutMs: normalizedOptions.timeout + 5000 }
+ : {},
+ );
if (res.status !== 200 || !res.data?.success) {
throwForBadResponse(res, "parse");
}
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts b/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts
index b6d8acda2..1ea236b85 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts
@@ -7,9 +7,16 @@ import {
} from "../types";
import { HttpClient } from "../utils/httpClient";
import { ensureValidScrapeOptions } from "../utils/validation";
-import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";
+import {
+ throwForBadResponse,
+ normalizeAxiosError,
+} from "../utils/errorHandler";
-export async function scrape(http: HttpClient, url: string, options?: ScrapeOptions): Promise<Document> {
+export async function scrape(
+ http: HttpClient,
+ url: string,
+ options?: ScrapeOptions,
+): Promise<Document> {
if (!url || !url.trim()) {
throw new Error("URL cannot be empty");
}
@@ -19,7 +26,17 @@ export async function scrape(http: HttpClient, url: string, options?: ScrapeOpti
if (options) Object.assign(payload, options);
try {
- const res = await http.post<{ success: boolean; data?: Document; error?: string }>("/v2/scrape", payload);
+ const res = await http.post<{
+ success: boolean;
+ data?: Document;
+ error?: string;
+ }>(
+ "/v2/scrape",
+ payload,
+ typeof options?.timeout === "number"
+ ? { timeoutMs: options.timeout + 5000 }
+ : {},
+ );
if (res.status !== 200 || !res.data?.success) {
throwForBadResponse(res, "scrape");
}
@@ -33,7 +50,7 @@ export async function scrape(http: HttpClient, url: string, options?: ScrapeOpti
export async function interact(
http: HttpClient,
jobId: string,
- args: ScrapeExecuteRequest
+ args: ScrapeExecuteRequest,
): Promise<ScrapeExecuteResponse> {
if (!jobId || !jobId.trim()) {
throw new Error("Job ID cannot be empty");
@@ -54,19 +71,22 @@ export async function interact(
try {
const res = await http.post<ScrapeExecuteResponse>(
`/v2/scrape/${jobId}/interact`,
- body
+ body,
+ args.timeout != null ? { timeoutMs: args.timeout * 1000 + 5000 } : {},
);
- if (res.status !== 200) throwForBadResponse(res, "interact with scrape browser");
+ if (res.status !== 200)
+ throwForBadResponse(res, "interact with scrape browser");
return res.data;
} catch (err: any) {
- if (err?.isAxiosError) return normalizeAxiosError(err, "interact with scrape browser");
+ if (err?.isAxiosError)
+ return normalizeAxiosError(err, "interact with scrape browser");
throw err;
}
}
export async function stopInteraction(
http: HttpClient,
- jobId: string
+ jobId: string,
): Promise<ScrapeBrowserDeleteResponse> {
if (!jobId || !jobId.trim()) {
throw new Error("Job ID cannot be empty");
@@ -74,7 +94,7 @@ export async function stopInteraction(
try {
const res = await http.delete<ScrapeBrowserDeleteResponse>(
- `/v2/scrape/${jobId}/interact`
+ `/v2/scrape/${jobId}/interact`,
);
if (res.status !== 200) throwForBadResponse(res, "stop interaction");
return res.data;
@@ -88,7 +108,7 @@ export async function stopInteraction(
export async function scrapeExecute(
http: HttpClient,
jobId: string,
- args: ScrapeExecuteRequest
+ args: ScrapeExecuteRequest,
): Promise<ScrapeExecuteResponse> {
return interact(http, jobId, args);
}
@@ -96,7 +116,7 @@ export async function scrapeExecute(
/** @deprecated Use stopInteraction(). */
export async function stopInteractiveBrowser(
http: HttpClient,
- jobId: string
+ jobId: string,
): Promise<ScrapeBrowserDeleteResponse> {
return stopInteraction(http, jobId);
}
@@ -104,8 +124,7 @@ export async function stopInteractiveBrowser(
/** @deprecated Use stopInteraction(). */
export async function deleteScrapeBrowser(
http: HttpClient,
- jobId: string
+ jobId: string,
): Promise<ScrapeBrowserDeleteResponse> {
return stopInteraction(http, jobId);
}
-
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
index 700594357..3abf53085 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
@@ -1,12 +1,25 @@
-import { type Document, type SearchData, type SearchRequest, type SearchResultWeb, type ScrapeOptions, type SearchResultNews, type SearchResultImages } from "../types";
+import {
+ type Document,
+ type SearchData,
+ type SearchRequest,
+ type SearchResultWeb,
+ type ScrapeOptions,
+ type SearchResultNews,
+ type SearchResultImages,
+} from "../types";
import { HttpClient } from "../utils/httpClient";
import { ensureValidScrapeOptions } from "../utils/validation";
-import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";
+import {
+ throwForBadResponse,
+ normalizeAxiosError,
+} from "../utils/errorHandler";
function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
- if (req.limit != null && req.limit <= 0) throw new Error("limit must be positive");
- if (req.timeout != null && req.timeout <= 0) throw new Error("timeout must be positive");
+ if (req.limit != null && req.limit <= 0)
+ throw new Error("limit must be positive");
+ if (req.timeout != null && req.timeout <= 0)
+ throw new Error("timeout must be positive");
const payload: Record<string, unknown> = {
query: req.query,
};
@@ -15,9 +28,11 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
if (req.limit != null) payload.limit = req.limit;
if (req.tbs != null) payload.tbs = req.tbs;
if (req.location != null) payload.location = req.location;
- if (req.ignoreInvalidURLs != null) payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
+ if (req.ignoreInvalidURLs != null)
+ payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
if (req.timeout != null) payload.timeout = req.timeout;
- if (req.integration && req.integration.trim()) payload.integration = req.integration.trim();
+ if (req.integration && req.integration.trim())
+ payload.integration = req.integration.trim();
if (req.origin) payload.origin = req.origin;
if (req.scrapeOptions) {
ensureValidScrapeOptions(req.scrapeOptions as ScrapeOptions);
@@ -51,10 +66,23 @@ function transformArray<ResultType>(arr: any[]): Array<ResultType | Document> {
return results;
}
-export async function search(http: HttpClient, request: SearchRequest): Promise<SearchData> {
+export async function search(
+ http: HttpClient,
+ request: SearchRequest,
+): Promise<SearchData> {
const payload = prepareSearchPayload(request);
try {
- const res = await http.post<{ success: boolean; data?: Record<string, unknown>; error?: string }>("/v2/search", payload);
+ const res = await http.post<{
+ success: boolean;
+ data?: Record<string, unknown>;
+ error?: string;
+ }>(
+ "/v2/search",
+ payload,
+ typeof request.timeout === "number"
+ ? { timeoutMs: request.timeout + 5000 }
+ : {},
+ );
if (res.status !== 200 || !res.data?.success) {
throwForBadResponse(res, "search");
}
@@ -62,11 +90,11 @@ export async function search(http: HttpClient, request: SearchRequest): Promise<
const out: SearchData = {};
if (data.web) out.web = transformArray<SearchResultWeb>(data.web);
if (data.news) out.news = transformArray<SearchResultNews>(data.news);
- if (data.images) out.images = transformArray<SearchResultImages>(data.images);
+ if (data.images)
+ out.images = transformArray<SearchResultImages>(data.images);
return out;
} catch (err: any) {
if (err?.isAxiosError) return normalizeAxiosError(err, "search");
throw err;
}
}
-
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
index eb1d38a42..5dc2cbbdb 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
@@ -1,4 +1,8 @@
-import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse } from "axios";
+import axios, {
+ type AxiosInstance,
+ type AxiosRequestConfig,
+ type AxiosResponse,
+} from "axios";
import { getVersion } from "./getVersion";
export interface HttpClientOptions {
@@ -9,6 +13,11 @@ export interface HttpClientOptions {
backoffFactor?: number; // seconds factor for 0.5, 1, 2...
}
+export interface RequestOptions {
+ headers?: Record<string, string>;
+ timeoutMs?: number;
+}
+
export class HttpClient {
private instance: AxiosInstance;
private readonly apiKey: string;
@@ -39,7 +48,9 @@ export class HttpClient {
return this.apiKey;
}
- private async request<T = any>(config: AxiosRequestConfig): Promise<AxiosResponse<T>> {
+ private async request<T = any>(
+ config: AxiosRequestConfig,
+ ): Promise<AxiosResponse<T>> {
const version = getVersion();
config.headers = {
...(config.headers || {}),
@@ -64,12 +75,13 @@ export class HttpClient {
["post", "put", "patch"].includes(cfg.method.toLowerCase())
) {
const data = (cfg.data ?? {}) as Record<string, unknown>;
- cfg.data = { ...data, origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${version}` };
-
- // If timeout is specified in the body, use it to override the request timeout
- if (typeof data.timeout === "number") {
- cfg.timeout = data.timeout + 5000;
- }
+ cfg.data = {
+ ...data,
+ origin:
+ typeof data.origin === "string" && data.origin.includes("mcp")
+ ? data.origin
+ : `js-sdk@${version}`,
+ };
}
if (isFormDataBody) {
@@ -98,25 +110,34 @@ export class HttpClient {
}
private sleep(seconds: number): Promise<void> {
- return new Promise((r) => setTimeout(r, seconds * 1000));
+ return new Promise(r => setTimeout(r, seconds * 1000));
}
- post<T = any>(endpoint: string, body: Record<string, unknown>, headers?: Record<string, string>) {
- return this.request<T>({ method: "post", url: endpoint, data: body, headers });
+ post<T = any>(
+ endpoint: string,
+ body: Record<string, unknown>,
+ options?: RequestOptions,
+ ) {
+ return this.request<T>({
+ method: "post",
+ url: endpoint,
+ data: body,
+ headers: options?.headers,
+ timeout: options?.timeoutMs,
+ });
}
postMultipart<T = any>(
endpoint: string,
formData: FormData,
- headers?: Record<string, string>,
- timeoutMs?: number,
+ options?: RequestOptions,
) {
return this.request<T>({
method: "post",
url: endpoint,
data: formData,
- headers,
- timeout: timeoutMs,
+ headers: options?.headers,
+ timeout: options?.timeoutMs,
});
}
@@ -134,4 +155,3 @@ export class HttpClient {
return headers;
}
}
-