Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(js-sdk): poll the batch scrape status endpoint in v1 batchScrapeUrls (#4918)

Gergő Móricz7bf45055dac5
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index abfea8474..351406a27 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.42.2",
+ "version": "4.42.3",
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v1/batch-scrape-poll.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/batch-scrape-poll.test.ts
new file mode 100644
index 000000000..f2247d15e
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/batch-scrape-poll.test.ts
@@ -0,0 +1,91 @@
+import { describe, test, expect, jest, afterEach, beforeEach } from "@jest/globals";
+import axios from "axios";
+import FirecrawlApp from "../../../v1";
+
+const API_URL = "https://api.firecrawl.dev";
+const API_KEY = "fc-test";
+
+const scrapingPage = () => ({ success: true, status: "scraping", total: 2, completed: 0, data: [] });
+const completedPage = (next: string) => ({
+ success: true,
+ status: "completed",
+ total: 2,
+ completed: 2,
+ expiresAt: "2030-01-01T00:00:00Z",
+ data: [{ markdown: "a" }],
+ next,
+});
+const lastPage = () => ({ success: true, status: "completed", data: [{ markdown: "b" }] });
+
+function mockPoll(path: string, nextOrigin = API_URL) {
+ const sent: string[] = [];
+ let polls = 0;
+ jest.spyOn(axios, "get").mockImplementation(async (url: string) => {
+ sent.push(url);
+ if (new URL(url).searchParams.has("skip")) return { status: 200, data: lastPage() };
+ polls++;
+ return { status: 200, data: polls === 1 ? scrapingPage() : completedPage(`${nextOrigin}${path}?skip=1`) };
+ });
+ return sent;
+}
+
+describe("v1 batch scrape waits on the batch scrape status endpoint", () => {
+ const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });
+
+ beforeEach(() => {
+ jest.spyOn(global, "setTimeout").mockImplementation(((fn: () => void) => {
+ fn();
+ return 0;
+ }) as any);
+ });
+
+ afterEach(() => {
+ jest.restoreAllMocks();
+ });
+
+ test("batchScrapeUrls polls /v1/batch/scrape/{id} and follows pinned next pages", async () => {
+ const post = jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
+ const path = "/v1/batch/scrape/abc";
+ const sent = mockPoll(path, "https://evil.example");
+
+ const res: any = await app.batchScrapeUrls(["https://example.com", "https://example.org"], {});
+
+ expect(post.mock.calls[0][0]).toBe(`${API_URL}/v1/batch/scrape`);
+ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
+ expect(sent.some(url => url.includes("/v1/crawl/"))).toBe(false);
+ expect(res.status).toBe("completed");
+ expect(res.data).toHaveLength(2);
+ });
+
+ test("batchScrapeUrls reports batch scrape failures as batch scrape errors", async () => {
+ jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });
+ const sent: string[] = [];
+ jest.spyOn(axios, "get").mockImplementation(async (url: string) => {
+ sent.push(url);
+ return { status: 200, data: { success: true, status: "failed", data: [] } };
+ });
+
+ await expect(app.batchScrapeUrls(["https://example.com"], {})).rejects.toThrow("Batch scrape job failed or was stopped");
+ expect(sent.every(url => url === `${API_URL}/v1/batch/scrape/abc`)).toBe(true);
+ });
+
+ test("monitorJobStatus still polls /v1/crawl/{id} by default", async () => {
+ const path = "/v1/crawl/abc";
+ const sent = mockPoll(path);
+
+ const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0);
+
+ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
+ expect(res.data).toHaveLength(2);
+ });
+
+ test("monitorJobStatus polls /v1/batch/scrape/{id} for batch jobs", async () => {
+ const path = "/v1/batch/scrape/abc";
+ const sent = mockPoll(path);
+
+ const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0, "batch");
+
+ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);
+ expect(res.data).toHaveLength(2);
+ });
+});
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
index 7a0f7bdd4..55f1d749d 100644
--- a/apps/js-sdk/firecrawl/src/v1/index.ts
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -1128,7 +1128,7 @@ export default class FirecrawlApp {
);
if (response.status === 200) {
const id: string = response.data.id;
- return this.monitorJobStatus(id, headers, pollInterval);
+ return this.monitorJobStatus(id, headers, pollInterval, "batch");
} else {
this.handleError(response, "start batch scrape job");
}
@@ -1484,18 +1484,21 @@ export default class FirecrawlApp {
}
/**
- * Monitors the status of a crawl job until completion or failure.
- * @param id - The ID of the crawl operation.
+ * Monitors the status of a crawl or batch scrape job until completion or failure.
+ * @param id - The ID of the crawl or batch scrape operation.
* @param headers - The headers for the request.
* @param checkInterval - Interval in seconds for job status checks.
- * @param checkUrl - Optional URL to check the status (used for v1 API)
+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.
* @returns The final job status or data.
*/
async monitorJobStatus(
id: string,
headers: AxiosRequestHeaders,
- checkInterval: number
+ checkInterval: number,
+ jobType: "crawl" | "batch" = "crawl"
): Promise<CrawlStatusResponse | ErrorResponse> {
+ const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";
+ const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";
let failedTries = 0;
let networkRetries = 0;
const maxNetworkRetries = 3;
@@ -1503,7 +1506,7 @@ export default class FirecrawlApp {
while (true) {
try {
let statusResponse: AxiosResponse = await this.getRequest(
- `${this.apiUrl}/v1/crawl/${id}`,
+ `${this.apiUrl}/v1/${statusPath}/${id}`,
headers
);
@@ -1526,7 +1529,7 @@ export default class FirecrawlApp {
statusData.data = data;
return statusData;
} else {
- throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+ throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);
}
} else if (
["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
@@ -1537,14 +1540,14 @@ export default class FirecrawlApp {
);
} else {
throw new FirecrawlError(
- `Crawl job failed or was stopped. Status: ${statusData.status}`,
+ `${jobLabel} job failed or was stopped. Status: ${statusData.status}`,
500
);
}
} else {
failedTries++;
if (failedTries >= 3) {
- this.handleError(statusResponse, "check crawl status");
+ this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);
}
}
} catch (error: any) {