Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(js-sdk): mostly done

rafaelmmillerd44baed8f282
diff --git a/apps/api/src/controllers/v2/token-usage.ts b/apps/api/src/controllers/v2/token-usage.ts
index 1e4f6116c..e99d176a8 100644
--- a/apps/api/src/controllers/v2/token-usage.ts
+++ b/apps/api/src/controllers/v2/token-usage.ts
@@ -14,7 +14,7 @@ export async function tokenUsageController(
res.json({
success: true,
data: {
- remaining_tokens: req.acuc.remaining_credits,
+ remainingTokens: req.acuc.remaining_credits,
},
});
return;
@@ -33,7 +33,7 @@ export async function tokenUsageController(
res.json({
success: true,
data: {
- remaining_tokens: chunk.remaining_credits,
+ remainingTokens: chunk.remaining_credits,
},
});
} catch (error) {
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 1f8dad10a..da662497e 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -14,6 +14,7 @@ import { scrapeStatusController } from "../controllers/v2/scrape-status";
import { creditUsageController } from "../controllers/v2/credit-usage";
import { tokenUsageController } from "../controllers/v2/token-usage";
import { crawlCancelController } from "../controllers/v2/crawl-cancel";
+import { concurrencyCheckController } from "../controllers/v2/concurrency-check";
import {
authMiddleware,
checkCreditsMiddleware,
@@ -129,3 +130,9 @@ v2Router.get(
authMiddleware(RateLimiterMode.ExtractStatus),
wrap(tokenUsageController),
);
+
+v2Router.get(
+ "/concurrency-check",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(concurrencyCheckController),
+);
\ No newline at end of file
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 3b38b1fd8..7127d37f4 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -16,7 +16,7 @@
"build": "tsup",
"build-and-publish": "npm run build && npm publish --access public",
"publish-beta": "npm run build && npm publish --access public --tag beta",
- "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/**/*.test.ts",
+ "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/*.test.ts --detectOpenHandles",
"test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
},
"repository": {
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts
new file mode 100644
index 000000000..ca9fa7570
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts
@@ -0,0 +1,70 @@
+/**
+ * E2E tests for v2 batch scrape (translated from Python tests)
+ */
+import Firecrawl from "../../../index";
+import { config } from "dotenv";
+import { describe, test, expect } from "@jest/globals";
+
+config();
+
+const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";
+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
+
+const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });
+
+describe("v2.batch e2e", () => {
+ test("batch scrape minimal (wait)", async () => {
+ const urls = [
+ "https://docs.firecrawl.dev",
+ "https://firecrawl.dev",
+ ];
+ const job = await client.batchScrape(urls, { options: { formats: ["markdown"] }, pollInterval: 1, timeout: 180 });
+ expect(["completed", "failed"]).toContain(job.status);
+ expect(job.completed).toBeGreaterThanOrEqual(0);
+ expect(job.total).toBeGreaterThanOrEqual(0);
+ expect(Array.isArray(job.data)).toBe(true);
+ }, 240_000);
+
+ test("start batch minimal and status", async () => {
+ const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
+ const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });
+ expect(typeof start.id).toBe("string");
+ expect(typeof start.url).toBe("string");
+ const status = await client.getBatchScrapeStatus(start.id);
+ expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
+ expect(status.total).toBeGreaterThanOrEqual(0);
+ }, 120_000);
+
+ test("wait batch with all params", async () => {
+ const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
+ const job = await client.batchScrape(urls, {
+ options: {
+ formats: [
+ "markdown",
+ { type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } },
+ { type: "changeTracking", prompt: "Track changes", modes: ["json"] },
+ ],
+ onlyMainContent: true,
+ mobile: false,
+ },
+ ignoreInvalidURLs: true,
+ maxConcurrency: 2,
+ zeroDataRetention: false,
+ pollInterval: 1,
+ timeout: 180,
+ });
+ expect(["completed", "failed", "cancelled"]).toContain(job.status);
+ expect(job.completed).toBeGreaterThanOrEqual(0);
+ expect(job.total).toBeGreaterThanOrEqual(0);
+ expect(Array.isArray(job.data)).toBe(true);
+ }, 300_000);
+
+ test("cancel batch", async () => {
+ const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
+ const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, maxConcurrency: 1 });
+ expect(typeof start.id).toBe("string");
+ const cancelled = await client.cancelBatchScrape(start.id);
+ expect(cancelled).toBe(true);
+ }, 120_000);
+});
+
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts
new file mode 100644
index 000000000..96b378c15
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts
@@ -0,0 +1,181 @@
+/**
+ * E2E tests for v2 crawl (translated from Python tests)
+ */
+import Firecrawl from "../../../index";
+import { config } from "dotenv";
+import { describe, test, expect } from "@jest/globals";
+
+config();
+
+const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";
+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
+
+const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });
+
+describe("v2.crawl e2e", () => {
+ if (!client) {
+ console.warn("Skipping v2.crawl e2e: FIRECRAWL_API_KEY not set");
+ }
+
+ test("start crawl minimal request", async () => {
+ if (!client) throw new Error();
+ const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
+ expect(typeof job.id).toBe("string");
+ expect(typeof job.url).toBe("string");
+ }, 90_000);
+
+ test("start crawl with options", async () => {
+ if (!client) throw new Error();
+ const job = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5, maxDiscoveryDepth: 2 });
+ expect(typeof job.id).toBe("string");
+ expect(typeof job.url).toBe("string");
+ }, 90_000);
+
+ test("start crawl with prompt", async () => {
+ if (!client) throw new Error();
+ const job = await client.startCrawl("https://firecrawl.dev", { prompt: "Extract all blog posts", limit: 3 });
+ expect(typeof job.id).toBe("string");
+ expect(typeof job.url).toBe("string");
+ }, 90_000);
+
+ test("get crawl status", async () => {
+ if (!client) throw new Error();
+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
+ const status = await client.getCrawlStatus(start.id);
+ expect(["scraping", "completed", "failed", "cancelled"]).toContain(status.status);
+ expect(status.completed).toBeGreaterThanOrEqual(0);
+ // next/expiresAt may be null/undefined depending on state; check shape
+ expect(Array.isArray(status.data)).toBe(true);
+ }, 120_000);
+
+ test("cancel crawl", async () => {
+ if (!client) throw new Error();
+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
+ const ok = await client.cancelCrawl(start.id);
+ expect(ok).toBe(true);
+ }, 120_000);
+
+ test("get crawl errors", async () => {
+ if (!client) throw new Error();
+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
+ const resp = await client.getCrawlErrors(start.id);
+ expect(resp).toHaveProperty("errors");
+ expect(resp).toHaveProperty("robotsBlocked");
+ expect(Array.isArray(resp.errors)).toBe(true);
+ expect(Array.isArray(resp.robotsBlocked)).toBe(true);
+ for (const e of resp.errors) {
+ expect(typeof e.id === "string" || e.id == null).toBe(true);
+ expect(typeof e.timestamp === "string" || e.timestamp == null).toBe(true);
+ expect(typeof e.url === "string" || e.url == null).toBe(true);
+ expect(typeof e.error === "string" || e.error == null).toBe(true);
+ }
+ }, 120_000);
+
+ test("get crawl errors with invalid id should throw", async () => {
+ if (!client) throw new Error();
+ await expect(client.getCrawlErrors("invalid-job-id-12345")).rejects.toThrow();
+ }, 60_000);
+
+ test("get active crawls", async () => {
+ if (!client) throw new Error();
+ const active = await client.getActiveCrawls();
+ expect(typeof active.success).toBe("boolean");
+ expect(Array.isArray(active.crawls)).toBe(true);
+ for (const c of active.crawls) {
+ expect(typeof c.id).toBe("string");
+ expect(typeof c.teamId).toBe("string");
+ expect(typeof c.url).toBe("string");
+ if (c.options != null) {
+ expect(typeof c.options === "object").toBe(true);
+ }
+ }
+ }, 90_000);
+
+ test("get active crawls with running crawl", async () => {
+ if (!client) throw new Error();
+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 5 });
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ const active = await client.getActiveCrawls();
+ expect(Array.isArray(active.crawls)).toBe(true);
+ const ids = active.crawls.map(c => c.id);
+ expect(ids.includes(start.id)).toBe(true);
+ await client.cancelCrawl(start.id);
+ }, 120_000);
+
+ test("crawl with wait", async () => {
+ if (!client) throw new Error();
+ const job = await client.crawl("https://docs.firecrawl.dev", { limit: 3, maxDiscoveryDepth: 2, pollInterval: 1, timeout: 120 });
+ expect(["completed", "failed"]).toContain(job.status);
+ expect(job.completed).toBeGreaterThanOrEqual(0);
+ expect(job.total).toBeGreaterThanOrEqual(0);
+ expect(Array.isArray(job.data)).toBe(true);
+ }, 180_000);
+
+ test("crawl with prompt and wait", async () => {
+ if (!client) throw new Error();
+ const job = await client.crawl("https://docs.firecrawl.dev", { prompt: "Extract all blog posts", limit: 3, pollInterval: 1, timeout: 120 });
+ expect(["completed", "failed"]).toContain(job.status);
+ expect(job.completed).toBeGreaterThanOrEqual(0);
+ expect(job.total).toBeGreaterThanOrEqual(0);
+ expect(Array.isArray(job.data)).toBe(true);
+ }, 180_000);
+
+ test("crawl with scrape options", async () => {
+ if (!client) throw new Error();
+ const job = await client.startCrawl("https://docs.firecrawl.dev", {
+ limit: 2,
+ scrapeOptions: { formats: ["markdown", "links"], onlyMainContent: false, mobile: true },
+ });
+ expect(typeof job.id).toBe("string");
+ }, 120_000);
+
+ test("crawl with json format object", async () => {
+ if (!client) throw new Error();
+ const job = await client.startCrawl("https://docs.firecrawl.dev", {
+ limit: 2,
+ scrapeOptions: { formats: [{ type: "json", prompt: "Extract page title", schema: { type: "object", properties: { title: { type: "string" } }, required: ["title"] } }] },
+ });
+ expect(typeof job.id).toBe("string");
+ }, 120_000);
+
+ test("crawl all parameters", async () => {
+ if (!client) throw new Error();
+ const job = await client.startCrawl("https://docs.firecrawl.dev", {
+ prompt: "Extract all blog posts and documentation",
+ includePaths: ["/blog/*", "/docs/*"],
+ excludePaths: ["/admin/*"],
+ maxDiscoveryDepth: 3,
+ ignoreSitemap: false,
+ ignoreQueryParameters: true,
+ limit: 5,
+ crawlEntireDomain: true,
+ allowExternalLinks: false,
+ allowSubdomains: true,
+ delay: 1,
+ maxConcurrency: 2,
+ webhook: "https://example.com/hook",
+ scrapeOptions: {
+ formats: ["markdown", "html"],
+ headers: { "User-Agent": "Test Bot" },
+ includeTags: ["h1", "h2"],
+ excludeTags: ["nav"],
+ onlyMainContent: false,
+ timeout: 15_000,
+ waitFor: 2000,
+ mobile: true,
+ skipTlsVerification: true,
+ removeBase64Images: false,
+ },
+ zeroDataRetention: false,
+ });
+ expect(typeof job.id).toBe("string");
+ }, 180_000);
+
+ test("crawl params preview", async () => {
+ if (!client) throw new Error();
+ const params = await client.crawlParamsPreview("https://docs.firecrawl.dev", "Extract all blog posts and documentation");
+ expect(params && typeof params === "object").toBe(true);
+ // Optional fields may or may not be present; just assert object shape
+ }, 60_000);
+});
+
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/extract.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/extract.test.ts
new file mode 100644
index 000000000..89e900c9c
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/extract.test.ts
@@ -0,0 +1,66 @@
+/**
+ * E2E tests for v2 extract (proxied to v1), translated from Python tests
+ */
+import Firecrawl from "../../../index";
+import { config } from "dotenv";
+import { describe, test, expect } from "@jest/globals";
+import { z } from "zod";
+
+config();
+
+const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";
+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
+
+const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });
+
+describe("v2.extract e2e", () => {
+ test("extract minimal with prompt", async () => {
+ const resp = await client.extract({ urls: ["https://docs.firecrawl.dev"], prompt: "Extract the main page title" });
+ expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
+ }, 120_000);
+
+ test("extract with schema", async () => {
+ const schema = {
+ type: "object",
+ properties: { title: { type: "string" } },
+ required: ["title"],
+ } as const;
+ const resp = await client.extract({
+ urls: ["https://docs.firecrawl.dev"],
+ schema,
+ prompt: "Extract the main page title",
+ showSources: true,
+ enableWebSearch: false,
+ });
+ expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
+ if ((resp as any).sources != null) {
+ expect(typeof (resp as any).sources).toBe("object");
+ }
+ if (resp.data != null) {
+ expect(typeof resp.data).toBe("object");
+ expect((resp.data as any).title).toBeTruthy();
+ }
+ }, 180_000);
+
+ test("extract with zod schema", async () => {
+ const schema = z.object({
+ title: z.string(),
+ });
+ const resp = await client.extract({
+ urls: ["https://docs.firecrawl.dev"],
+ schema: schema,
+ prompt: "Extract the main page title",
+ showSources: true,
+ enableWebSearch: false,
+ });
+ expect(typeof resp.success === "boolean" || resp.success == null).toBe(true);
+ if ((resp as any).sources != null) {
+ expect(typeof (resp as any).sources).toBe("object");
+ }
+ if (resp.data != null) {
+ expect(typeof resp.data).toBe("object");
+ expect(schema.safeParse(resp.data).success).toBe(true);
+ }
+ }, 180_000);
+});
+
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/map.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/map.test.ts
new file mode 100644
index 000000000..551ca6847
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/map.test.ts
@@ -0,0 +1,54 @@
+/**
+ * E2E tests for v2 map (translated from Python tests)
+ */
+import Firecrawl from "../../../index";
+import { config } from "dotenv";
+import { describe, test, expect } from "@jest/globals";
+
+config();
+
+const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";
+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
+
+const client = API_KEY ? new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL }) : null as any;
+
+describe("v2.map e2e", () => {
+ if (!client) {
+ console.warn("Skipping v2.map e2e: FIRECRAWL_API_KEY not set");
+ }
+
+ test("minimal request", async () => {
+ if (!client) throw new Error();
+ const resp = await client.map("https://docs.firecrawl.dev");
+
+ expect(resp).toBeTruthy();
+ expect(Array.isArray(resp.links)).toBe(true);
+
+ if (resp.links.length > 0) {
+ const first: any = resp.links[0];
+ expect(typeof first.url).toBe("string");
+ expect(first.url.startsWith("http")).toBe(true);
+ }
+ }, 90_000);
+
+ test.each(["only", "skip", "include"]) ("with options sitemap=%s", async (sitemap) => {
+ if (!client) throw new Error();
+ const resp = await client.map("https://docs.firecrawl.dev", {
+ search: "docs",
+ includeSubdomains: true,
+ limit: 10,
+ sitemap: sitemap as any,
+ timeout: 15_000,
+ });
+
+ expect(resp).toBeTruthy();
+ expect(Array.isArray(resp.links)).toBe(true);
+ expect(resp.links.length).toBeLessThanOrEqual(10);
+
+ for (const link of resp.links as any[]) {
+ expect(typeof link.url).toBe("string");
+ expect(link.url.startsWith("http")).toBe(true);
+ }
+ }, 120_000);
+});
+
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.ts
similarity index 98%
rename from apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.ts
rename to apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.ts
index 3b3752896..a8d3b835d 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.ts
@@ -1,7 +1,7 @@
/**
* E2E tests for v2 scrape
*/
-import Firecrawl from "../../../../index";
+import Firecrawl from "../../../index";
import { z } from "zod";
import { config } from "dotenv";
import { describe, test, expect } from "@jest/globals";
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.ts
similarity index 96%
rename from apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.ts
rename to apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.ts
index 80871eea5..5b0228b8a 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.ts
@@ -1,7 +1,7 @@
/**
* E2E tests for v2 search (translated from Python tests)
*/
-import Firecrawl from "../../../../index";
+import Firecrawl from "../../../index";
import { config } from "dotenv";
import { describe, test, expect } from "@jest/globals";
@@ -171,7 +171,12 @@ describe("v2.search e2e", () => {
}
}
- expect(results.images == null).toBe(true);
+ expect(results.images).toBeTruthy();
+ expect((results.images || []).length).toBeLessThanOrEqual(3);
+ for (const result of results.images || []) {
+ expect(typeof result.url).toBe("string");
+ expect(result.url.startsWith("http")).toBe(true);
+ }
}, 120_000);
test("formats flexibility: list vs object", async () => {
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts
new file mode 100644
index 000000000..1767b5368
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts
@@ -0,0 +1,32 @@
+/**
+ * E2E tests for v2 usage endpoints (translated from Python tests)
+ */
+import Firecrawl from "../../../index";
+import { config } from "dotenv";
+import { describe, test, expect } from "@jest/globals";
+
+config();
+
+const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";
+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
+
+const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });
+
+describe("v2.usage e2e", () => {
+ test("get_concurrency", async () => {
+ const resp = await client.getConcurrency();
+ expect(typeof resp.concurrency).toBe("number");
+ expect(typeof resp.maxConcurrency).toBe("number");
+ }, 60_000);
+
+ test("get_credit_usage", async () => {
+ const resp = await client.getCreditUsage();
+ expect(typeof resp.remaining_credits).toBe("number");
+ }, 60_000);
+
+ test("get_token_usage", async () => {
+ const resp = await client.getTokenUsage();
+ expect(typeof resp.remainingTokens).toBe("number");
+ }, 60_000);
+});
+
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/watcher.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/watcher.test.ts
new file mode 100644
index 000000000..5f49b9128
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/watcher.test.ts
@@ -0,0 +1,90 @@
+import Firecrawl from "../../../index";
+import { config } from "dotenv";
+import { describe, test, expect } from "@jest/globals";
+
+config();
+
+const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";
+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
+
+describe("v2.watcher e2e", () => {
+ test("crawl watcher minimal", async () => {
+ const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });
+ const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
+
+ expect(typeof start.id).toBe("string");
+
+ const watcher = client.watcher(start.id, { pollInterval: 2 });
+
+ let snapshots = 0;
+ let documents = 0;
+
+ watcher.on("snapshot", (snap: any) => {
+ snapshots += 1;
+ expect(["scraping", "completed", "failed", "cancelled"]).toContain(snap.status);
+ expect(typeof snap.completed).toBe("number");
+ expect(typeof snap.total).toBe("number");
+ });
+
+ watcher.on("document", (_doc: any) => {
+ documents += 1;
+ });
+
+ const final = await new Promise<any>(async (resolve) => {
+ watcher.on("done", (payload: any) => {
+ resolve(payload);
+ });
+ watcher.on("error", (err: any) => {
+ resolve(err);
+ });
+ await watcher.start();
+ });
+
+ expect(["completed", "failed", "cancelled"]).toContain(final.status);
+ expect(Array.isArray(final.data)).toBe(true);
+ expect(typeof final.id).toBe("string");
+ expect(snapshots).toBeGreaterThanOrEqual(1);
+ expect(documents).toBeGreaterThanOrEqual(0);
+ watcher.close();
+ }, 240_000);
+
+ test("batch watcher with options (kind, pollInterval, timeout)", async () => {
+ const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });
+ const urls = [
+ "https://docs.firecrawl.dev",
+ "https://firecrawl.dev",
+ ];
+
+ const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] }, ignoreInvalidURLs: true });
+ expect(typeof start.id).toBe("string");
+
+ const watcher = client.watcher(start.id, { kind: "batch", pollInterval: 2, timeout: 180 });
+
+ let snapshots = 0;
+ let gotCompleted = false;
+
+ watcher.on("snapshot", (snap: any) => {
+ snapshots += 1;
+ if (snap.status === "completed") gotCompleted = true;
+ expect(["scraping", "completed", "failed", "cancelled"]).toContain(snap.status);
+ });
+
+ const final = await new Promise<any>(async (resolve) => {
+ watcher.on("done", (payload: any) => {
+ resolve(payload);
+ });
+ watcher.on("error", (err: any) => {
+ resolve(err);
+ });
+ await watcher.start();
+ });
+
+ expect(["completed", "failed", "cancelled"]).toContain(final.status);
+ expect(Array.isArray(final.data)).toBe(true);
+ expect(typeof final.id).toBe("string");
+ expect(snapshots).toBeGreaterThanOrEqual(1);
+ expect(gotCompleted || final.status !== "completed").toBe(true);
+ watcher.close();
+ }, 300_000);
+});
+
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts
new file mode 100644
index 000000000..f1b04f086
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/errorHandler.test.ts
@@ -0,0 +1,19 @@
+import { describe, test, expect } from "@jest/globals";
+import { throwForBadResponse, normalizeAxiosError } from "../../../v2/utils/errorHandler";
+
+describe("v2 utils: errorHandler", () => {
+ test("throwForBadResponse: throws SdkError with message from body.error", () => {
+ const resp: any = { status: 400, data: { error: "bad" } };
+ expect(() => throwForBadResponse(resp, "do thing")).toThrow(/bad/);
+ });
+
+ test("normalizeAxiosError: prefers body.error then err.message", () => {
+ const err: any = {
+ isAxiosError: true,
+ response: { status: 402, data: { error: "payment required" } },
+ message: "network",
+ };
+ expect(() => normalizeAxiosError(err, "action")).toThrow(/payment required/);
+ });
+});
+
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts
new file mode 100644
index 000000000..32aa895e6
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts
@@ -0,0 +1,59 @@
+import { describe, test, expect } from "@jest/globals";
+import { ensureValidFormats, ensureValidScrapeOptions } from "../../../v2/utils/validation";
+import type { FormatOption } from "../../../v2/types";
+import { z } from "zod";
+
+describe("v2 utils: validation", () => {
+ test("ensureValidFormats: plain 'json' string is invalid", () => {
+ const formats: FormatOption[] = ["markdown", "json"] as unknown as FormatOption[];
+ expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);
+ });
+
+ test("ensureValidFormats: json format requires prompt and schema", () => {
+ const bad1: FormatOption[] = [{ type: "json", prompt: "p" } as any];
+ const bad2: FormatOption[] = [{ type: "json", schema: {} } as any];
+ expect(() => ensureValidFormats(bad1)).toThrow(/requires 'prompt' and 'schema'/i);
+ expect(() => ensureValidFormats(bad2)).toThrow(/requires 'prompt' and 'schema'/i);
+ });
+
+ test("ensureValidFormats: converts zod schema to JSON schema", () => {
+ const schema = z.object({ title: z.string() });
+ const formats: FormatOption[] = [
+ { type: "json", prompt: "extract", schema } as any,
+ ];
+ ensureValidFormats(formats);
+ const jsonFmt = formats[0] as any;
+ expect(typeof jsonFmt.schema).toBe("object");
+ expect(jsonFmt.schema?.properties).toBeTruthy();
+ });
+
+ test("ensureValidFormats: screenshot quality must be non-negative number", () => {
+ const formats: FormatOption[] = [
+ { type: "screenshot", quality: -1 } as any,
+ ];
+ expect(() => ensureValidFormats(formats)).toThrow(/non-negative number/i);
+ });
+
+ test("ensureValidScrapeOptions: validates timeout and waitFor bounds", () => {
+ expect(() => ensureValidScrapeOptions({ timeout: 0 })).toThrow(/timeout must be positive/i);
+ expect(() => ensureValidScrapeOptions({ waitFor: -1 })).toThrow(/waitFor must be non-negative/i);
+ // valid
+ expect(() => ensureValidScrapeOptions({ timeout: 1000, waitFor: 0 })).not.toThrow();
+ });
+
+ test("ensureValidFormats: accepts screenshot viewport width/height", () => {
+ const formats: FormatOption[] = [
+ { type: "screenshot", viewport: { width: 800, height: 600 } } as any,
+ ];
+ expect(() => ensureValidFormats(formats)).not.toThrow();
+ expect((formats[0] as any).viewport).toEqual({ width: 800, height: 600 });
+ });
+
+ test("ensureValidScrapeOptions: leaves parsers untouched", () => {
+ const options = { parsers: ["pdf", "images"] as string[] } as any;
+ const before = [...options.parsers];
+ expect(() => ensureValidScrapeOptions(options)).not.toThrow();
+ expect(options.parsers).toEqual(before);
+ });
+});
+
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
index 74a6bfd4c..c8df2ebb5 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
@@ -132,9 +132,9 @@ export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise
export async function getActiveCrawls(http: HttpClient): Promise<ActiveCrawlsResponse> {
try {
- const res = await http.get<{ success: boolean; data?: { crawls: Array<{ id: string; teamId?: string; team_id?: string; url: string; options?: any }> } }>(`/v2/crawl/active`);
+ const res = await http.get<{ success: boolean; crawls: Array<{ id: string; teamId?: string; team_id?: string; url: string; options?: any }> }>(`/v2/crawl/active`);
if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get active crawls");
- const crawlsIn = res.data?.data?.crawls || [];
+ const crawlsIn = res.data?.crawls || [];
const crawls = crawlsIn.map((c) => ({ id: c.id, teamId: (c as any).teamId ?? (c as any).team_id, url: c.url, options: c.options ?? null }));
return { success: true, crawls };
} catch (err: any) {
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/extract.ts b/apps/js-sdk/firecrawl/src/v2/methods/extract.ts
index ebcd2f52d..7b6cd7e18 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/extract.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/extract.ts
@@ -2,11 +2,13 @@ import { type ExtractResponse, type ScrapeOptions } from "../types";
import { HttpClient } from "../utils/httpClient";
import { ensureValidScrapeOptions } from "../utils/validation";
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+import { zodToJsonSchema } from "zod-to-json-schema";
+import type { ZodTypeAny } from "zod";
function prepareExtractPayload(args: {
urls?: string[];
prompt?: string;
- schema?: Record<string, unknown>;
+ schema?: Record<string, unknown> | ZodTypeAny;
systemPrompt?: string;
allowExternalLinks?: boolean;
enableWebSearch?: boolean;
@@ -17,7 +19,11 @@ function prepareExtractPayload(args: {
const body: Record<string, unknown> = {};
if (args.urls) body.urls = args.urls;
if (args.prompt != null) body.prompt = args.prompt;
- if (args.schema != null) body.schema = args.schema;
+ if (args.schema != null) {
+ const s: any = args.schema;
+ const isZod = s && (typeof s.safeParse === "function" || typeof s.parse === "function") && s._def;
+ body.schema = isZod ? zodToJsonSchema(s) : args.schema;
+ }
if (args.systemPrompt != null) body.systemPrompt = args.systemPrompt;
if (args.allowExternalLinks != null) body.allowExternalLinks = args.allowExternalLinks;
if (args.enableWebSearch != null) body.enableWebSearch = args.enableWebSearch;
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/usage.ts b/apps/js-sdk/firecrawl/src/v2/methods/usage.ts
index 4138a70ec..dfd79d5b3 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/usage.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/usage.ts
@@ -16,7 +16,7 @@ export async function getConcurrency(http: HttpClient): Promise<ConcurrencyCheck
export async function getCreditUsage(http: HttpClient): Promise<CreditUsage> {
try {
- const res = await http.get<{ success: boolean; data?: { remaining_credits: number } }>("/v2/team/credit-usage");
+ const res = await http.get<{ success: boolean; data?: { remaining_credits: number } }>("/v2/credit-usage");
if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get credit usage");
const d = res.data.data || (res.data as any);
return { remaining_credits: d.remaining_credits };
@@ -28,7 +28,7 @@ export async function getCreditUsage(http: HttpClient): Promise<CreditUsage> {
export async function getTokenUsage(http: HttpClient): Promise<TokenUsage> {
try {
- const res = await http.get<{ success: boolean; data?: TokenUsage }>("/v2/team/token-usage");
+ const res = await http.get<{ success: boolean; data?: TokenUsage }>("/v2/token-usage");
if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get token usage");
return (res.data.data || (res.data as any)) as TokenUsage;
} catch (err: any) {
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 5dccff574..74d9f1654 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -259,11 +259,7 @@ export interface CreditUsage {
}
export interface TokenUsage {
- promptTokens: number;
- completionTokens: number;
- totalTokens: number;
- step?: string;
- model?: string;
+ remainingTokens: number;
}
export interface CrawlErrorsResponse {
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts
index b646e9316..d1df63538 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts
@@ -1,10 +1,14 @@
export function getVersion(): string {
try {
+ if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
+ return process.env.npm_package_version as string;
+ }
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
- const pkg = require("../../package.json");
- return (pkg?.version as string) || "3.0.0";
+ const pkg = require("../../../package.json");
+ return (pkg?.version as string) || "3.x.x";
} catch {
- return "3.0.0";
+ return "3.x.x";
}
}
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
index b2e57b487..96914517c 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
@@ -48,8 +48,6 @@ export class HttpClient {
origin: `js-sdk@${version}`,
};
- console.log("request", config);
-
let lastError: any;
for (let attempt = 0; attempt < this.maxRetries; attempt++) {
try {
diff --git a/apps/js-sdk/firecrawl/src/v2/watcher.ts b/apps/js-sdk/firecrawl/src/v2/watcher.ts
index 2e595c20d..f64e610f9 100644
--- a/apps/js-sdk/firecrawl/src/v2/watcher.ts
+++ b/apps/js-sdk/firecrawl/src/v2/watcher.ts
@@ -123,7 +123,10 @@ export class Watcher extends EventEmitter {
data,
};
this.emit("snapshot", snap);
- if (["completed", "failed", "cancelled"].includes(status)) this.emit("done", { status, data, id: this.jobId });
+ if (["completed", "failed", "cancelled"].includes(status)) {
+ this.emit("done", { status, data, id: this.jobId });
+ this.close();
+ }
}
private async pollLoop() {
@@ -131,10 +134,13 @@ export class Watcher extends EventEmitter {
const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;
while (!this.closed) {
try {
- const snap = this.kind === "crawl" ? await getCrawlStatus(this as any as HttpClient, this.jobId) : await getBatchScrapeStatus(this as any as HttpClient, this.jobId);
+ const snap = this.kind === "crawl"
+ ? await getCrawlStatus(this.http as any, this.jobId)
+ : await getBatchScrapeStatus(this.http as any, this.jobId);
this.emit("snapshot", snap);
if (["completed", "failed", "cancelled"].includes(snap.status)) {
this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });
+ this.close();
break;
}
} catch {
diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py
index b6b2ee1ac..2873a4eae 100644
--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py
@@ -455,8 +455,7 @@ def get_active_crawls(client: HttpClient) -> ActiveCrawlsResponse:
if not body.get("success"):
raise Exception(body.get("error", "Unknown error occurred"))
- data = body.get("data", {})
- crawls_in = data.get("crawls", [])
+ crawls_in = body.get("crawls", [])
normalized_crawls = []
for c in crawls_in:
if isinstance(c, dict):
diff --git a/apps/python-sdk/firecrawl/v2/methods/usage.py b/apps/python-sdk/firecrawl/v2/methods/usage.py
index a210d5979..b014045b9 100644
--- a/apps/python-sdk/firecrawl/v2/methods/usage.py
+++ b/apps/python-sdk/firecrawl/v2/methods/usage.py
@@ -17,7 +17,7 @@ def get_concurrency(client: HttpClient) -> ConcurrencyCheck:
def get_credit_usage(client: HttpClient) -> CreditUsage:
- resp = client.get("/v2/team/credit-usage")
+ resp = client.get("/v2/credit-usage")
if not resp.ok:
handle_response_error(resp, "get credit usage")
body = resp.json()
@@ -28,7 +28,7 @@ def get_credit_usage(client: HttpClient) -> CreditUsage:
def get_token_usage(client: HttpClient) -> TokenUsage:
- resp = client.get("/v2/team/token-usage")
+ resp = client.get("/v2/token-usage")
if not resp.ok:
handle_response_error(resp, "get token usage")
body = resp.json()