Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(js-sdk): poll the batch scrape status endpoint in v1 batchScrapeUrls (#4918)
Gergő Móricz7bf45055dac5
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex abfea8474..351406a27 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.42.2",+ "version": "4.42.3","description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v1/batch-scrape-poll.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/batch-scrape-poll.test.tsnew file mode 100644index 000000000..f2247d15e--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/batch-scrape-poll.test.ts@@ -0,0 +1,91 @@+import { describe, test, expect, jest, afterEach, beforeEach } from "@jest/globals";+import axios from "axios";+import FirecrawlApp from "../../../v1";++const API_URL = "https://api.firecrawl.dev";+const API_KEY = "fc-test";++const scrapingPage = () => ({ success: true, status: "scraping", total: 2, completed: 0, data: [] });+const completedPage = (next: string) => ({+ success: true,+ status: "completed",+ total: 2,+ completed: 2,+ expiresAt: "2030-01-01T00:00:00Z",+ data: [{ markdown: "a" }],+ next,+});+const lastPage = () => ({ success: true, status: "completed", data: [{ markdown: "b" }] });++function mockPoll(path: string, nextOrigin = API_URL) {+ const sent: string[] = [];+ let polls = 0;+ jest.spyOn(axios, "get").mockImplementation(async (url: string) => {+ sent.push(url);+ if (new URL(url).searchParams.has("skip")) return { status: 200, data: lastPage() };+ polls++;+ return { status: 200, data: polls === 1 ? scrapingPage() : completedPage(`${nextOrigin}${path}?skip=1`) };+ });+ return sent;+}++describe("v1 batch scrape waits on the batch scrape status endpoint", () => {+ const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });++ beforeEach(() => {+ jest.spyOn(global, "setTimeout").mockImplementation(((fn: () => void) => {+ fn();+ return 0;+ }) as any);+ });++ afterEach(() => {+ jest.restoreAllMocks();+ });++ test("batchScrapeUrls polls /v1/batch/scrape/{id} and follows pinned next pages", async () => {+ const post = jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });+ const path = "/v1/batch/scrape/abc";+ const sent = mockPoll(path, "https://evil.example");++ const res: any = await app.batchScrapeUrls(["https://example.com", "https://example.org"], {});++ expect(post.mock.calls[0][0]).toBe(`${API_URL}/v1/batch/scrape`);+ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);+ expect(sent.some(url => url.includes("/v1/crawl/"))).toBe(false);+ expect(res.status).toBe("completed");+ expect(res.data).toHaveLength(2);+ });++ test("batchScrapeUrls reports batch scrape failures as batch scrape errors", async () => {+ jest.spyOn(axios, "post").mockResolvedValue({ status: 200, data: { success: true, id: "abc" } });+ const sent: string[] = [];+ jest.spyOn(axios, "get").mockImplementation(async (url: string) => {+ sent.push(url);+ return { status: 200, data: { success: true, status: "failed", data: [] } };+ });++ await expect(app.batchScrapeUrls(["https://example.com"], {})).rejects.toThrow("Batch scrape job failed or was stopped");+ expect(sent.every(url => url === `${API_URL}/v1/batch/scrape/abc`)).toBe(true);+ });++ test("monitorJobStatus still polls /v1/crawl/{id} by default", async () => {+ const path = "/v1/crawl/abc";+ const sent = mockPoll(path);++ const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0);++ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);+ expect(res.data).toHaveLength(2);+ });++ test("monitorJobStatus polls /v1/batch/scrape/{id} for batch jobs", async () => {+ const path = "/v1/batch/scrape/abc";+ const sent = mockPoll(path);++ const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0, "batch");++ expect(sent).toEqual([`${API_URL}${path}`, `${API_URL}${path}`, `${API_URL}${path}?skip=1`]);+ expect(res.data).toHaveLength(2);+ });+});diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex 7a0f7bdd4..55f1d749d 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -1128,7 +1128,7 @@ export default class FirecrawlApp {);if (response.status === 200) {const id: string = response.data.id;- return this.monitorJobStatus(id, headers, pollInterval);+ return this.monitorJobStatus(id, headers, pollInterval, "batch");} else {this.handleError(response, "start batch scrape job");}@@ -1484,18 +1484,21 @@ export default class FirecrawlApp {}/**- * Monitors the status of a crawl job until completion or failure.- * @param id - The ID of the crawl operation.+ * Monitors the status of a crawl or batch scrape job until completion or failure.+ * @param id - The ID of the crawl or batch scrape operation.* @param headers - The headers for the request.* @param checkInterval - Interval in seconds for job status checks.- * @param checkUrl - Optional URL to check the status (used for v1 API)+ * @param jobType - Which status endpoint to poll. Defaults to `"crawl"`.* @returns The final job status or data.*/async monitorJobStatus(id: string,headers: AxiosRequestHeaders,- checkInterval: number+ checkInterval: number,+ jobType: "crawl" | "batch" = "crawl"): Promise<CrawlStatusResponse | ErrorResponse> {+ const statusPath = jobType === "batch" ? "batch/scrape" : "crawl";+ const jobLabel = jobType === "batch" ? "Batch scrape" : "Crawl";let failedTries = 0;let networkRetries = 0;const maxNetworkRetries = 3;@@ -1503,7 +1506,7 @@ export default class FirecrawlApp {while (true) {try {let statusResponse: AxiosResponse = await this.getRequest(- `${this.apiUrl}/v1/crawl/${id}`,+ `${this.apiUrl}/v1/${statusPath}/${id}`,headers);@@ -1526,7 +1529,7 @@ export default class FirecrawlApp {statusData.data = data;return statusData;} else {- throw new FirecrawlError("Crawl job completed but no data was returned", 500);+ throw new FirecrawlError(`${jobLabel} job completed but no data was returned`, 500);}} else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)@@ -1537,14 +1540,14 @@ export default class FirecrawlApp {);} else {throw new FirecrawlError(- `Crawl job failed or was stopped. Status: ${statusData.status}`,+ `${jobLabel} job failed or was stopped. Status: ${statusData.status}`,500);}} else {failedTries++;if (failedTries >= 3) {- this.handleError(statusResponse, "check crawl status");+ this.handleError(statusResponse, `check ${jobLabel.toLowerCase()} status`);}}} catch (error: any) {