Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(js-sdk): mostly done
rafaelmmillerd44baed8f282
diff --git a/apps/api/src/controllers/v2/token-usage.ts b/apps/api/src/controllers/v2/token-usage.tsindex 1e4f6116c..e99d176a8 100644--- a/apps/api/src/controllers/v2/token-usage.ts+++ b/apps/api/src/controllers/v2/token-usage.ts@@ -14,7 +14,7 @@ export async function tokenUsageController(res.json({success: true,data: {- remaining_tokens: req.acuc.remaining_credits,+ remainingTokens: req.acuc.remaining_credits,},});return;@@ -33,7 +33,7 @@ export async function tokenUsageController(res.json({success: true,data: {- remaining_tokens: chunk.remaining_credits,+ remainingTokens: chunk.remaining_credits,},});} catch (error) {diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 1f8dad10a..da662497e 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -14,6 +14,7 @@ import { scrapeStatusController } from "../controllers/v2/scrape-status";import { creditUsageController } from "../controllers/v2/credit-usage";import { tokenUsageController } from "../controllers/v2/token-usage";import { crawlCancelController } from "../controllers/v2/crawl-cancel";+import { concurrencyCheckController } from "../controllers/v2/concurrency-check";import {authMiddleware,checkCreditsMiddleware,@@ -129,3 +130,9 @@ v2Router.get(authMiddleware(RateLimiterMode.ExtractStatus),wrap(tokenUsageController),);++v2Router.get(+ "/concurrency-check",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(concurrencyCheckController),+);\ No newline at end of filediff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 3b38b1fd8..7127d37f4 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -16,7 +16,7 @@"build": "tsup","build-and-publish": "npm run build && npm publish --access public","publish-beta": "npm run build && npm publish --access public --tag beta",- "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/**/*.test.ts",+ "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles","test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"},"repository": {diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.tsnew file mode 100644index 000000000..ca9fa7570--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts@@ -0,0 +1,70 @@+/**+ * E2E tests for v2 batch scrape (translated from Python tests)+ */+import Firecrawl from "../../../index";+import { config } from "dotenv";+import { describe, test, expect } from "@jest/globals";++config();++const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";++const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });++describe("v2.batch e2e", () => {+ test("batch scrape minimal (wait)", async () => {+ const urls = [+ "https://docs.firecrawl.dev",+ "https://firecrawl.dev",+ ];+ const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });+ expect(["completed", "failed"]).toContain(job.status);+ expect(job.completed).toBeGreaterThanOrEqual(0);+ expect(job.total).toBeGreaterThanOrEqual(0);+ expect(Array.isArray(job.data)).toBe(true);+ }, 240_000);++ test("start batch minimal and status", async () => {+ const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];+ const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });+ expect(typeof start.id).toBe("string");+ expect(typeof start.url).toBe("string");+ const status = await client.getBatchScrapeStatus(start.id);+ expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);+ expect(status.total).toBeGreaterThanOrEqual(0);+ }, 120_000);++ test("wait batch with all params", async () => {+ const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];+ const job = await client.batchScrape(urls, {+ options: {+ formats: [+ "markdown",+ { type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } },+ { type: "changeTracking", prompt: "Track changes", modes: ["json"] },+ ],+ onlyMainContent: true,+ mobile: false,+ },+ ignoreInvalidURLs: true,+ maxConcurrency: 2,+ zeroDataRetention: false,+ pollInterval: 1,+ timeout: 180,+ });+ expect(["completed", "failed", "cancelled"]).toContain(job.status);+ expect(job.completed).toBeGreaterThanOrEqual(0);+ expect(job.total).toBeGreaterThanOrEqual(0);+ expect(Array.isArray(job.data)).toBe(true);+ }, 300_000);++ test("cancel batch", async () => {+ const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];+ const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, maxConcurrency: 1 });+ expect(typeof start.id).toBe("string");+ const cancelled = await client.cancelBatchScrape(start.id);+ expect(cancelled).toBe(true);+ }, 120_000);+});+diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.tsnew file mode 100644index 000000000..96b378c15--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts@@ -0,0 +1,181 @@+/**+ * E2E tests for v2 crawl (translated from Python tests)+ */+import Firecrawl from "../../../index";+import { config } from "dotenv";+import { describe, test, expect } from "@jest/globals";++config();++const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";++const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });++describe("v2.crawl e2e", () => {+ if (!client) {+ console.warn("Skipping v2.crawl e2e: FIRECRAWL_API_KEY not set");+ }++ test("start crawl minimal request", async () => {+ if (!client) throw new Error();+ const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });+ expect(typeof job.id).toBe("string");+ expect(typeof job.url).toBe("string");+ }, 90_000);++ test("start crawl with options", async () => {+ if (!client) throw new Error();+ const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5, maxDiscoveryDepth: 2 });+ expect(typeof job.id).toBe("string");+ expect(typeof job.url).toBe("string");+ }, 90_000);++ test("start crawl with prompt", async () => {+ if (!client) throw new Error();+ const job = await client.startCrawl("https://firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });+ expect(typeof job.id).toBe("string");+ expect(typeof job.url).toBe("string");+ }, 90_000);++ test("get crawl status", async () => {+ if (!client) throw new Error();+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });+ const status = await client.getCrawlStatus(start.id);+ expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);+ expect(status.completed).toBeGreaterThanOrEqual(0);+ // next/expiresAt may be null/undefined depending on state; check shape+ expect(Array.isArray(status.data)).toBe(true);+ }, 120_000);++ test("cancel crawl", async () => {+ if (!client) throw new Error();+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });+ const ok = await client.cancelCrawl(start.id);+ expect(ok).toBe(true);+ }, 120_000);++ test("get crawl errors", async () => {+ if (!client) throw new Error();+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });+ const resp = await client.getCrawlErrors(start.id);+ expect(resp).toHaveProperty("errors");+ expect(resp).toHaveProperty("robotsBlocked");+ expect(Array.isArray(resp.errors)).toBe(true);+ expect(Array.isArray(resp.robotsBlocked)).toBe(true);+ for (const e of resp.errors) {+ expect(typeof e.id === "string" || e.id == null).toBe(true);+ expect(typeof e.timestamp === "string" || e.timestamp == null).toBe(true);+ expect(typeof e.url === "string" || e.url == null).toBe(true);+ expect(typeof e.error === "string" || e.error == null).toBe(true);+ }+ }, 120_000);++ test("get crawl errors with invalid id should throw", async () => {+ if (!client) throw new Error();+ await expect(client.getCrawlErrors("invalid-job-id-12345")).rejects.toThrow();+ }, 60_000);++ test("get active crawls", async () => {+ if (!client) throw new Error();+ const active = await client.getActiveCrawls();+ expect(typeof active.success).toBe("boolean");+ expect(Array.isArray(active.crawls)).toBe(true);+ for (const c of active.crawls) {+ expect(typeof c.id).toBe("string");+ expect(typeof c.teamId).toBe("string");+ expect(typeof c.url).toBe("string");+ if (c.options != null) {+ expect(typeof c.options === "object").toBe(true);+ }+ }+ }, 90_000);++ test("get active crawls with running crawl", async () => {+ if (!client) throw new Error();+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5 });+ await new Promise(resolve => setTimeout(resolve, 1000));+ const active = await client.getActiveCrawls();+ expect(Array.isArray(active.crawls)).toBe(true);+ const ids = active.crawls.map(c => c.id);+ expect(ids.includes(start.id)).toBe(true);+ await client.cancelCrawl(start.id);+ }, 120_000);++ test("crawl with wait", async () => {+ if (!client) throw new Error();+ const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });+ expect(["completed", "failed"]).toContain(job.status);+ expect(job.completed).toBeGreaterThanOrEqual(0);+ expect(job.total).toBeGreaterThanOrEqual(0);+ expect(Array.isArray(job.data)).toBe(true);+ }, 180_000);++ test("crawl with prompt and wait", async () => {+ if (!client) throw new Error();+ const job = await client.crawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3, pollInterval: 1, timeout: 120 });+ expect(["completed", "failed"]).toContain(job.status);+ expect(job.completed).toBeGreaterThanOrEqual(0);+ expect(job.total).toBeGreaterThanOrEqual(0);+ expect(Array.isArray(job.data)).toBe(true);+ }, 180_000);++ test("crawl with scrape options", async () => {+ if (!client) throw new Error();+ const job = await client.startCrawl("https://docs.firecrawl.dev", {+ limit: 2,+ scrapeOptions: { formats: ["markdown", "links"], onlyMainContent: false, mobile: true },+ });+ expect(typeof job.id).toBe("string");+ }, 120_000);++ test("crawl with json format object", async () => {+ if (!client) throw new Error();+ const job = await client.startCrawl("https://docs.firecrawl.dev", {+ limit: 2,+ scrapeOptions: { formats: [{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } }] },+ });+ expect(typeof job.id).toBe("string");+ }, 120_000);++ test("crawl all parameters", async () => {+ if (!client) throw new Error();+ const job = await client.startCrawl("https://docs.firecrawl.dev", {+ prompt: "Extract all blog posts and documentation",+ includePaths: ["/blog/*", "/docs/*"],+ excludePaths: ["/admin/*"],+ maxDiscoveryDepth: 3,+ ignoreSitemap: false,+ ignoreQueryParameters: true,+ limit: 5,+ crawlEntireDomain: true,+ allowExternalLinks: false,+ allowSubdomains: true,+ delay: 1,+ maxConcurrency: 2,+ webhook: "https://example.com/hook",+ scrapeOptions: {+ formats: ["markdown", "html"],+ headers: { "User-Agent": "Test Bot" },+ includeTags: ["h1", "h2"],+ excludeTags: ["nav"],+ onlyMainContent: false,+ timeout: 15_000,+ waitFor: 2000,+ mobile: true,+ skipTlsVerification: true,+ removeBase64Images: false,+ },+ zeroDataRetention: false,+ });+ expect(typeof job.id).toBe("string");+ }, 180_000);++ test("crawl params preview", async () => {+ if (!client) throw new Error();+ const params = await client.crawlParamsPreview("https://docs.firecrawl.dev", "Extract all blog posts and documentation");+ expect(params && typeof params === "object").toBe(true);+ // Optional fields may or may not be present; just assert object shape+ }, 60_000);+});+diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/extract.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/extract.test.tsnew file mode 100644index 000000000..89e900c9c--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/extract.test.ts@@ -0,0 +1,66 @@+/**+ * E2E tests for v2 extract (proxied to v1), translated from Python tests+ */+import Firecrawl from "../../../index";+import { config } from "dotenv";+import { describe, test, expect } from "@jest/globals";+import { z } from "zod";++config();++const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";++const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });++describe("v2.extract e2e", () => {+ test("extract minimal with prompt", async () => {+ const resp = await client.extract({ urls: ["https://docs.firecrawl.dev"], prompt: "Extract the main page title" });+ expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);+ }, 120_000);++ test("extract with schema", async () => {+ const schema = {+ type: "object",+ properties: { title: { type: "string" } },+ required: ["title"],+ } as const;+ const resp = await client.extract({+ urls: ["https://docs.firecrawl.dev"],+ schema,+ prompt: "Extract the main page title",+ showSources: true,+ enableWebSearch: false,+ });+ expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);+ if ((resp as any).sources != null) {+ expect(typeof (resp as any).sources).toBe("object");+ }+ if (resp.data != null) {+ expect(typeof resp.data).toBe("object");+ expect((resp.data as any).title).toBeTruthy();+ }+ }, 180_000);++ test("extract with zod schema", async () => {+ const schema = z.object({+ title: z.string(),+ });+ const resp = await client.extract({+ urls: ["https://docs.firecrawl.dev"],+ schema: schema,+ prompt: "Extract the main page title",+ showSources: true,+ enableWebSearch: false,+ });+ expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);+ if ((resp as any).sources != null) {+ expect(typeof (resp as any).sources).toBe("object");+ }+ if (resp.data != null) {+ expect(typeof resp.data).toBe("object");+ expect(schema.safeParse(resp.data).success).toBe(true);+ }+ }, 180_000);+});+diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/map.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/map.test.tsnew file mode 100644index 000000000..551ca6847--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/map.test.ts@@ -0,0 +1,54 @@+/**+ * E2E tests for v2 map (translated from Python tests)+ */+import Firecrawl from "../../../index";+import { config } from "dotenv";+import { describe, test, expect } from "@jest/globals";++config();++const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";++const client = API_KEY ? new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL }) : null as any;++describe("v2.map e2e", () => {+ if (!client) {+ console.warn("Skipping v2.map e2e: FIRECRAWL_API_KEY not set");+ }++ test("minimal request", async () => {+ if (!client) throw new Error();+ const resp = await client.map("https://docs.firecrawl.dev");++ expect(resp).toBeTruthy();+ expect(Array.isArray(resp.links)).toBe(true);++ if (resp.links.length > 0) {+ const first: any = resp.links[0];+ expect(typeof first.url).toBe("string");+ expect(first.url.startsWith("http")).toBe(true);+ }+ }, 90_000);++ test.each(["only", "skip", "include"]) ("with options sitemap=%s", async (sitemap) => {+ if (!client) throw new Error();+ const resp = await client.map("https://docs.firecrawl.dev", {+ search: "docs",+ includeSubdomains: true,+ limit: 10,+ sitemap: sitemap as any,+ timeout: 15_000,+ });++ expect(resp).toBeTruthy();+ expect(Array.isArray(resp.links)).toBe(true);+ expect(resp.links.length).toBeLessThanOrEqual(10);++ for (const link of resp.links as any[]) {+ expect(typeof link.url).toBe("string");+ expect(link.url.startsWith("http")).toBe(true);+ }+ }, 120_000);+});+diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.tssimilarity index 98%rename from apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.tsrename to apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.tsindex 3b3752896..a8d3b835d 100644--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.ts@@ -1,7 +1,7 @@/*** E2E tests for v2 scrape*/-import Firecrawl from "../../../../index";+import Firecrawl from "../../../index";import { z } from "zod";import { config } from "dotenv";import { describe, test, expect } from "@jest/globals";diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.tssimilarity index 96%rename from apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.tsrename to apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.tsindex 80871eea5..5b0228b8a 100644--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.ts@@ -1,7 +1,7 @@/*** E2E tests for v2 search (translated from Python tests)*/-import Firecrawl from "../../../../index";+import Firecrawl from "../../../index";import { config } from "dotenv";import { describe, test, expect } from "@jest/globals";@@ -171,7 +171,12 @@ describe("v2.search e2e", () => {}}- expect(results.images == null).toBe(true);+ expect(results.images).toBeTruthy();+ expect((results.images || []).length).toBeLessThanOrEqual(3);+ for (const result of results.images || []) {+ expect(typeof result.url).toBe("string");+ expect(result.url.startsWith("http")).toBe(true);+ }}, 120_000);test("formats flexibility: list vs object", async () => {diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.tsnew file mode 100644index 000000000..1767b5368--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts@@ -0,0 +1,32 @@+/**+ * E2E tests for v2 usage endpoints (translated from Python tests)+ */+import Firecrawl from "../../../index";+import { config } from "dotenv";+import { describe, test, expect } from "@jest/globals";++config();++const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";++const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });++describe("v2.usage e2e", () => {+ test("get_concurrency", async () => {+ const resp = await client.getConcurrency();+ expect(typeof resp.concurrency).toBe("number");+ expect(typeof resp.maxConcurrency).toBe("number");+ }, 60_000);++ test("get_credit_usage", async () => {+ const resp = await client.getCreditUsage();+ expect(typeof resp.remaining_credits).toBe("number");+ }, 60_000);++ test("get_token_usage", async () => {+ const resp = await client.getTokenUsage();+ expect(typeof resp.remainingTokens).toBe("number");+ }, 60_000);+});+diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/watcher.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/watcher.test.tsnew file mode 100644index 000000000..5f49b9128--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/watcher.test.ts@@ -0,0 +1,90 @@+import Firecrawl from "../../../index";+import { config } from "dotenv";+import { describe, test, expect } from "@jest/globals";++config();++const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";++describe("v2.watcher e2e", () => {+ test("crawl watcher minimal", async () => {+ const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });++ expect(typeof start.id).toBe("string");++ const watcher = client.watcher(start.id, { pollInterval: 2 });++ let snapshots = 0;+ let documents = 0;++ watcher.on("snapshot", (snap: any) => {+ snapshots += 1;+ expect(["scraping", "completed", "failed", "cancelled"]).toContain(snap.status);+ expect(typeof snap.completed).toBe("number");+ expect(typeof snap.total).toBe("number");+ });++ watcher.on("document", (_doc: any) => {+ documents += 1;+ });++ const final = await new Promise<any>(async (resolve) => {+ watcher.on("done", (payload: any) => {+ resolve(payload);+ });+ watcher.on("error", (err: any) => {+ resolve(err);+ });+ await watcher.start();+ });++ expect(["completed", "failed", "cancelled"]).toContain(final.status);+ expect(Array.isArray(final.data)).toBe(true);+ expect(typeof final.id).toBe("string");+ expect(snapshots).toBeGreaterThanOrEqual(1);+ expect(documents).toBeGreaterThanOrEqual(0);+ watcher.close();+ }, 240_000);++ test("batch watcher with options (kind, pollInterval, timeout)", async () => {+ const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });+ const urls = [+ "https://docs.firecrawl.dev",+ "https://firecrawl.dev",+ ];++ const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });+ expect(typeof start.id).toBe("string");++ const watcher = client.watcher(start.id, { kind: "batch", pollInterval: 2, timeout: 180 });++ let snapshots = 0;+ let gotCompleted = false;++ watcher.on("snapshot", (snap: any) => {+ snapshots += 1;+ if (snap.status === "completed") gotCompleted = true;+ expect(["scraping", "completed", "failed", "cancelled"]).toContain(snap.status);+ });++ const final = await new Promise<any>(async (resolve) => {+ watcher.on("done", (payload: any) => {+ resolve(payload);+ });+ watcher.on("error", (err: any) => {+ resolve(err);+ });+ await watcher.start();+ });++ expect(["completed", "failed", "cancelled"]).toContain(final.status);+ expect(Array.isArray(final.data)).toBe(true);+ expect(typeof final.id).toBe("string");+ expect(snapshots).toBeGreaterThanOrEqual(1);+ expect(gotCompleted || final.status !== "completed").toBe(true);+ watcher.close();+ }, 300_000);+});+diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.tsnew file mode 100644index 000000000..f1b04f086--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts@@ -0,0 +1,19 @@+import { describe, test, expect } from "@jest/globals";+import { throwForBadResponse, normalizeAxiosError } from "../../../v2/utils/errorHandler";++describe("v2 utils: errorHandler", () => {+ test("throwForBadResponse: throws SdkError with message from body.error", () => {+ const resp: any = { status: 400, data: { error: "bad" } };+ expect(() => throwForBadResponse(resp, "do thing")).toThrow(/bad/);+ });++ test("normalizeAxiosError: prefers body.error then err.message", () => {+ const err: any = {+ isAxiosError: true,+ response: { status: 402, data: { error: "payment required" } },+ message: "network",+ };+ expect(() => normalizeAxiosError(err, "action")).toThrow(/payment required/);+ });+});+diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.tsnew file mode 100644index 000000000..32aa895e6--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts@@ -0,0 +1,59 @@+import { describe, test, expect } from "@jest/globals";+import { ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";+import type { FormatOption } from "../../../v2/types";+import { z } from "zod";++describe("v2 utils: validation", () => {+ test("ensureValidFormats: plain 'json' string is invalid", () => {+ const formats: FormatOption[] = ["markdown", "json"] as unknown as FormatOption[];+ expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);+ });++ test("ensureValidFormats: json format requires prompt and schema", () => {+ const bad1: FormatOption[] = [{ type: "json", prompt: "p" } as any];+ const bad2: FormatOption[] = [{ type: "json", schema: {} } as any];+ expect(() => ensureValidFormats(bad1)).toThrow(/requires 'prompt' and 'schema'/i);+ expect(() => ensureValidFormats(bad2)).toThrow(/requires 'prompt' and 'schema'/i);+ });++ test("ensureValidFormats: converts zod schema to JSON schema", () => {+ const schema = z.object({ title: z.string() });+ const formats: FormatOption[] = [+ { type: "json", prompt: "extract", schema } as any,+ ];+ ensureValidFormats(formats);+ const jsonFmt = formats[0] as any;+ expect(typeof jsonFmt.schema).toBe("object");+ expect(jsonFmt.schema?.properties).toBeTruthy();+ });++ test("ensureValidFormats: screenshot quality must be non-negative number", () => {+ const formats: FormatOption[] = [+ { type: "screenshot", quality: -1 } as any,+ ];+ expect(() => ensureValidFormats(formats)).toThrow(/non-negative number/i);+ });++ test("ensureValidScrapeOptions: validates timeout and waitFor bounds", () => {+ expect(() => ensureValidScrapeOptions({ timeout: 0 })).toThrow(/timeout must be positive/i);+ expect(() => ensureValidScrapeOptions({ waitFor: -1 })).toThrow(/waitFor must be non-negative/i);+ // valid+ expect(() => ensureValidScrapeOptions({ timeout: 1000, waitFor: 0 })).not.toThrow();+ });++ test("ensureValidFormats: accepts screenshot viewport width/height", () => {+ const formats: FormatOption[] = [+ { type: "screenshot", viewport: { width: 800, height: 600 } } as any,+ ];+ expect(() => ensureValidFormats(formats)).not.toThrow();+ expect((formats[0] as any).viewport).toEqual({ width: 800, height: 600 });+ });++ test("ensureValidScrapeOptions: leaves parsers untouched", () => {+ const options = { parsers: ["pdf", "images"] as string[] } as any;+ const before = [...options.parsers];+ expect(() => ensureValidScrapeOptions(options)).not.toThrow();+ expect(options.parsers).toEqual(before);+ });+});+diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.tsindex 74a6bfd4c..c8df2ebb5 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts@@ -132,9 +132,9 @@ export async function getCrawlErrors(http: HttpClient, crawlId: string): Promiseexport async function getActiveCrawls(http: HttpClient): Promise<ActiveCrawlsResponse> {try {- const res = await http.get<{ success: boolean; data?: { crawls: Array<{ id: string; teamId?: string; team_id?: string; url: string; options?: any }> } }>(`/v2/crawl/active`);+ const res = await http.get<{ success: boolean; crawls: Array<{ id: string; teamId?: string; team_id?: string; url: string; options?: any }> }>(`/v2/crawl/active`);if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get active crawls");- const crawlsIn = res.data?.data?.crawls || [];+ const crawlsIn = res.data?.crawls || [];const crawls = crawlsIn.map((c) => ({ id: c.id, teamId: (c as any).teamId ?? (c as any).team_id, url: c.url, options: c.options ?? null }));return { success: true, crawls };} catch (err: any) {diff --git a/apps/js-sdk/firecrawl/src/v2/methods/extract.ts b/apps/js-sdk/firecrawl/src/v2/methods/extract.tsindex ebcd2f52d..7b6cd7e18 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/extract.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/extract.ts@@ -2,11 +2,13 @@ import { type ExtractResponse, type ScrapeOptions } from "../types";import { HttpClient } from "../utils/httpClient";import { ensureValidScrapeOptions } from "../utils/validation";import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";+import { zodToJsonSchema } from "zod-to-json-schema";+import type { ZodTypeAny } from "zod";function prepareExtractPayload(args: {urls?: string[];prompt?: string;- schema?: Record<string, unknown>;+ schema?: Record<string, unknown> | ZodTypeAny;systemPrompt?: string;allowExternalLinks?: boolean;enableWebSearch?: boolean;@@ -17,7 +19,11 @@ function prepareExtractPayload(args: {const body: Record<string, unknown> = {};if (args.urls) body.urls = args.urls;if (args.prompt != null) body.prompt = args.prompt;- if (args.schema != null) body.schema = args.schema;+ if (args.schema != null) {+ const s: any = args.schema;+ const isZod = s && (typeof s.safeParse === "function" || typeof s.parse === "function") && s._def;+ body.schema = isZod ? zodToJsonSchema(s) : args.schema;+ }if (args.systemPrompt != null) body.systemPrompt = args.systemPrompt;if (args.allowExternalLinks != null) body.allowExternalLinks = args.allowExternalLinks;if (args.enableWebSearch != null) body.enableWebSearch = args.enableWebSearch;diff --git a/apps/js-sdk/firecrawl/src/v2/methods/usage.ts b/apps/js-sdk/firecrawl/src/v2/methods/usage.tsindex 4138a70ec..dfd79d5b3 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/usage.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/usage.ts@@ -16,7 +16,7 @@ export async function getConcurrency(http: HttpClient): Promise<ConcurrencyCheckexport async function getCreditUsage(http: HttpClient): Promise<CreditUsage> {try {- const res = await http.get<{ success: boolean; data?: { remaining_credits: number } }>("/v2/team/credit-usage");+ const res = await http.get<{ success: boolean; data?: { remaining_credits: number } }>("/v2/credit-usage");if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get credit usage");const d = res.data.data || (res.data as any);return { remaining_credits: d.remaining_credits };@@ -28,7 +28,7 @@ export async function getCreditUsage(http: HttpClient): Promise<CreditUsage> {export async function getTokenUsage(http: HttpClient): Promise<TokenUsage> {try {- const res = await http.get<{ success: boolean; data?: TokenUsage }>("/v2/team/token-usage");+ const res = await http.get<{ success: boolean; data?: TokenUsage }>("/v2/token-usage");if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get token usage");return (res.data.data || (res.data as any)) as TokenUsage;} catch (err: any) {diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 5dccff574..74d9f1654 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -259,11 +259,7 @@ export interface CreditUsage {}export interface TokenUsage {- promptTokens: number;- completionTokens: number;- totalTokens: number;- step?: string;- model?: string;+ remainingTokens: number;}export interface CrawlErrorsResponse {diff --git a/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.tsindex b646e9316..d1df63538 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts@@ -1,10 +1,14 @@export function getVersion(): string {try {+ if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {+ return process.env.npm_package_version as string;+ }+// eslint-disable-next-line @typescript-eslint/no-var-requires- const pkg = require("../../package.json");- return (pkg?.version as string) || "3.0.0";+ const pkg = require("../../../package.json");+ return (pkg?.version as string) || "3.x.x";} catch {- return "3.0.0";+ return "3.x.x";}}diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.tsindex b2e57b487..96914517c 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts@@ -48,8 +48,6 @@ export class HttpClient {origin: `js-sdk@${version}`,};- console.log("request", config);-let lastError: any;for (let attempt = 0; attempt < this.maxRetries; attempt++) {try {diff --git a/apps/js-sdk/firecrawl/src/v2/watcher.ts b/apps/js-sdk/firecrawl/src/v2/watcher.tsindex 2e595c20d..f64e610f9 100644--- a/apps/js-sdk/firecrawl/src/v2/watcher.ts+++ b/apps/js-sdk/firecrawl/src/v2/watcher.ts@@ -123,7 +123,10 @@ export class Watcher extends EventEmitter {data,};this.emit("snapshot", snap);- if (["completed", "failed", "cancelled"].includes(status)) this.emit("done", { status, data, id: this.jobId });+ if (["completed", "failed", "cancelled"].includes(status)) {+ this.emit("done", { status, data, id: this.jobId });+ this.close();+ }}private async pollLoop() {@@ -131,10 +134,13 @@ export class Watcher extends EventEmitter {const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;while (!this.closed) {try {- const snap = this.kind === "crawl" ? await getCrawlStatus(this as any as HttpClient, this.jobId) : await getBatchScrapeStatus(this as any as HttpClient, this.jobId);+ const snap = this.kind === "crawl"+ ? await getCrawlStatus(this.http as any, this.jobId)+ : await getBatchScrapeStatus(this.http as any, this.jobId);this.emit("snapshot", snap);if (["completed", "failed", "cancelled"].includes(snap.status)) {this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });+ this.close();break;}} catch {diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex b6b2ee1ac..2873a4eae 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -455,8 +455,7 @@ def get_active_crawls(client: HttpClient) -> ActiveCrawlsResponse:if not body.get("success"):raise Exception(body.get("error", "Unknown error occurred"))- data = body.get("data", {})- crawls_in = data.get("crawls", [])+ crawls_in = body.get("crawls", [])normalized_crawls = []for c in crawls_in:if isinstance(c, dict):diff --git a/apps/python-sdk/firecrawl/v2/methods/usage.py b/apps/python-sdk/firecrawl/v2/methods/usage.pyindex a210d5979..b014045b9 100644--- a/apps/python-sdk/firecrawl/v2/methods/usage.py+++ b/apps/python-sdk/firecrawl/v2/methods/usage.py@@ -17,7 +17,7 @@ def get_concurrency(client: HttpClient) -> ConcurrencyCheck:def get_credit_usage(client: HttpClient) -> CreditUsage:- resp = client.get("/v2/team/credit-usage")+ resp = client.get("/v2/credit-usage")if not resp.ok:handle_response_error(resp, "get credit usage")body = resp.json()@@ -28,7 +28,7 @@ def get_credit_usage(client: HttpClient) -> CreditUsage:def get_token_usage(client: HttpClient) -> TokenUsage:- resp = client.get("/v2/team/token-usage")+ resp = client.get("/v2/token-usage")if not resp.ok:handle_response_error(resp, "get token usage")body = resp.json()