Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(js-sdk): Return job ID from synchronous methods (#2414)
Abimael Martellcce38a8639a6
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 8deaa5081..8e87043c4 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.5.1",+ "version": "4.6.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.tsindex 72168a5ce..df09e1746 100644--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts@@ -29,6 +29,23 @@ describe("v2.batch e2e", () => {expect(Array.isArray(job.data)).toBe(true);}, 240_000);+ test("batch scrape with wait returns job id for error retrieval", async () => {+ const urls = [+ "https://docs.firecrawl.dev",+ "https://firecrawl.dev",+ ];+ const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });+ // Verify job has id field+ expect(job.id).toBeDefined();+ expect(typeof job.id).toBe("string");+ // Verify we can use the id to retrieve errors+ const errors = await client.getBatchScrapeErrors(job.id!);+ expect(errors).toHaveProperty("errors");+ expect(errors).toHaveProperty("robotsBlocked");+ expect(Array.isArray(errors.errors)).toBe(true);+ expect(Array.isArray(errors.robotsBlocked)).toBe(true);+ }, 240_000);+test("start batch minimal and status", async () => {const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });@@ -37,6 +54,9 @@ describe("v2.batch e2e", () => {const status = await client.getBatchScrapeStatus(start.id);expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);expect(status.total).toBeGreaterThanOrEqual(0);+ // Verify status includes id field+ expect(status.id).toBeDefined();+ expect(status.id).toBe(start.id);}, 120_000);test("wait batch with all params", async () => {diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.tsindex 2f2283ea1..97f36b5d8 100644--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts@@ -45,6 +45,9 @@ describe("v2.crawl e2e", () => {const status = await client.getCrawlStatus(start.id);expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);expect(status.completed).toBeGreaterThanOrEqual(0);+ // Verify status includes id field+ expect(status.id).toBeDefined();+ expect(status.id).toBe(start.id);// next/expiresAt may be null/undefined depending on state; check shapeexpect(Array.isArray(status.data)).toBe(true);}, 120_000);@@ -112,6 +115,20 @@ describe("v2.crawl e2e", () => {expect(Array.isArray(job.data)).toBe(true);}, 180_000);+ test("crawl with wait returns job id for error retrieval", async () => {+ if (!client) throw new Error();+ const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });+ // Verify job has id field+ expect(job.id).toBeDefined();+ expect(typeof job.id).toBe("string");+ // Verify we can use the id to retrieve errors+ const errors = await client.getCrawlErrors(job.id!);+ expect(errors).toHaveProperty("errors");+ expect(errors).toHaveProperty("robotsBlocked");+ expect(Array.isArray(errors.errors)).toBe(true);+ expect(Array.isArray(errors.robotsBlocked)).toBe(true);+ }, 180_000);+test("crawl with prompt and wait", async () => {if (!client) throw new Error();const job = await client.crawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3, pollInterval: 1, timeout: 120 });diff --git a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts b/apps/js-sdk/firecrawl/src/v2/methods/batch.tsindex 06050fed7..684736f8f 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts@@ -62,6 +62,7 @@ export async function getBatchScrapeStatus(const auto = pagination?.autoPaginate ?? true;if (!auto || !body.next) {return {+ id: jobId,status: body.status,completed: body.completed ?? 0,total: body.total ?? 0,@@ -74,6 +75,7 @@ export async function getBatchScrapeStatus(const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);return {+ id: jobId,status: body.status,completed: body.completed ?? 0,total: body.total ?? 0,@@ -136,4 +138,4 @@ export function chunkUrls(urls: string[], chunkSize = 100): string[][] {const chunks: string[][] = [];for (let i = 0; i < urls.length; i += chunkSize) chunks.push(urls.slice(i, i + chunkSize));return chunks;-}\ No newline at end of file+}diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.tsindex 33b5b58fe..b9201fb20 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts@@ -72,6 +72,7 @@ export async function getCrawlStatus(const auto = pagination?.autoPaginate ?? true;if (!auto || !body.next) {return {+ id: jobId,status: body.status,completed: body.completed ?? 0,total: body.total ?? 0,@@ -85,6 +86,7 @@ export async function getCrawlStatus(const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);return {+ id: jobId,status: body.status,completed: body.completed ?? 0,total: body.total ?? 0,@@ -165,4 +167,4 @@ export async function crawlParamsPreview(http: HttpClient, url: string, prompt:if (err?.isAxiosError) return normalizeAxiosError(err, "crawl params preview");throw err;}-}\ No newline at end of file+}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex fb7733c84..f32483e53 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -466,6 +466,7 @@ export interface CrawlResponse {}export interface CrawlJob {+ id: string;status: 'scraping' | 'completed' | 'failed' | 'cancelled';total: number;completed: number;@@ -493,6 +494,7 @@ export interface BatchScrapeResponse {}export interface BatchScrapeJob {+ id: string;status: 'scraping' | 'completed' | 'failed' | 'cancelled';completed: number;total: number;