Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(js-sdk): Return job ID from synchronous methods (#2414)

Abimael Martellcce38a8639a6
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 8deaa5081..8e87043c4 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.5.1",
+ "version": "4.6.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts
index 72168a5ce..df09e1746 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts
@@ -29,6 +29,23 @@ describe("v2.batch e2e", () => {
expect(Array.isArray(job.data)).toBe(true);
}, 240_000);
+ test("batch scrape with wait returns job id for error retrieval", async () => {
+ const urls = [
+ "https://docs.firecrawl.dev",
+ "https://firecrawl.dev",
+ ];
+ const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
+ // Verify job has id field
+ expect(job.id).toBeDefined();
+ expect(typeof job.id).toBe("string");
+ // Verify we can use the id to retrieve errors
+ const errors = await client.getBatchScrapeErrors(job.id!);
+ expect(errors).toHaveProperty("errors");
+ expect(errors).toHaveProperty("robotsBlocked");
+ expect(Array.isArray(errors.errors)).toBe(true);
+ expect(Array.isArray(errors.robotsBlocked)).toBe(true);
+ }, 240_000);
+
test("start batch minimal and status", async () => {
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });
@@ -37,6 +54,9 @@ describe("v2.batch e2e", () => {
const status = await client.getBatchScrapeStatus(start.id);
expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
expect(status.total).toBeGreaterThanOrEqual(0);
+ // Verify status includes id field
+ expect(status.id).toBeDefined();
+ expect(status.id).toBe(start.id);
}, 120_000);
test("wait batch with all params", async () => {
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts
index 2f2283ea1..97f36b5d8 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts
@@ -45,6 +45,9 @@ describe("v2.crawl e2e", () => {
const status = await client.getCrawlStatus(start.id);
expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
expect(status.completed).toBeGreaterThanOrEqual(0);
+ // Verify status includes id field
+ expect(status.id).toBeDefined();
+ expect(status.id).toBe(start.id);
// next/expiresAt may be null/undefined depending on state; check shape
expect(Array.isArray(status.data)).toBe(true);
}, 120_000);
@@ -112,6 +115,20 @@ describe("v2.crawl e2e", () => {
expect(Array.isArray(job.data)).toBe(true);
}, 180_000);
+ test("crawl with wait returns job id for error retrieval", async () => {
+ if (!client) throw new Error();
+ const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
+ // Verify job has id field
+ expect(job.id).toBeDefined();
+ expect(typeof job.id).toBe("string");
+ // Verify we can use the id to retrieve errors
+ const errors = await client.getCrawlErrors(job.id!);
+ expect(errors).toHaveProperty("errors");
+ expect(errors).toHaveProperty("robotsBlocked");
+ expect(Array.isArray(errors.errors)).toBe(true);
+ expect(Array.isArray(errors.robotsBlocked)).toBe(true);
+ }, 180_000);
+
test("crawl with prompt and wait", async () => {
if (!client) throw new Error();
const job = await client.crawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3, pollInterval: 1, timeout: 120 });
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
index 06050fed7..684736f8f 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
@@ -62,6 +62,7 @@ export async function getBatchScrapeStatus(
const auto = pagination?.autoPaginate ?? true;
if (!auto || !body.next) {
return {
+ id: jobId,
status: body.status,
completed: body.completed ?? 0,
total: body.total ?? 0,
@@ -74,6 +75,7 @@ export async function getBatchScrapeStatus(
const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);
return {
+ id: jobId,
status: body.status,
completed: body.completed ?? 0,
total: body.total ?? 0,
@@ -136,4 +138,4 @@ export function chunkUrls(urls: string[], chunkSize = 100): string[][] {
const chunks: string[][] = [];
for (let i = 0; i < urls.length; i += chunkSize) chunks.push(urls.slice(i, i + chunkSize));
return chunks;
-}
\ No newline at end of file
+}
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
index 33b5b58fe..b9201fb20 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
@@ -72,6 +72,7 @@ export async function getCrawlStatus(
const auto = pagination?.autoPaginate ?? true;
if (!auto || !body.next) {
return {
+ id: jobId,
status: body.status,
completed: body.completed ?? 0,
total: body.total ?? 0,
@@ -85,6 +86,7 @@ export async function getCrawlStatus(
const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);
return {
+ id: jobId,
status: body.status,
completed: body.completed ?? 0,
total: body.total ?? 0,
@@ -165,4 +167,4 @@ export async function crawlParamsPreview(http: HttpClient, url: string, prompt:
if (err?.isAxiosError) return normalizeAxiosError(err, "crawl params preview");
throw err;
}
-}
\ No newline at end of file
+}
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index fb7733c84..f32483e53 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -466,6 +466,7 @@ export interface CrawlResponse {
}
export interface CrawlJob {
+ id: string;
status: 'scraping' | 'completed' | 'failed' | 'cancelled';
total: number;
completed: number;
@@ -493,6 +494,7 @@ export interface BatchScrapeResponse {
}
export interface BatchScrapeJob {
+ id: string;
status: 'scraping' | 'completed' | 'failed' | 'cancelled';
completed: number;
total: number;