Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(js-sdk): methods done. todo: e2e/unit tests

rafaelmmillerec69c309920b
diff --git a/apps/js-sdk/.env.example b/apps/js-sdk/.env.example
new file mode 100644
index 000000000..81394d178
--- /dev/null
+++ b/apps/js-sdk/.env.example
@@ -0,0 +1,2 @@
+FIRECRAWL_API_KEY=<your api key>
+FIRECRAWL_API_URL=https://api.firecrawl.dev
\ No newline at end of file
diff --git a/apps/js-sdk/example.js b/apps/js-sdk/example.js
index 21bdb2a1b..d473743f4 100644
--- a/apps/js-sdk/example.js
+++ b/apps/js-sdk/example.js
@@ -1,3 +1,38 @@
+// Placeholder v2 example (JavaScript)
+// Minimal usage of new FirecrawlClient. Replace with your API key before running.
+
+import { FirecrawlClient } from './firecrawl/src/v2/client.js';
+
+async function run() {
+ const apiKey = (globalThis.process && globalThis.process.env && globalThis.process.env.FIRECRAWL_API_KEY) || 'fc-YOUR_API_KEY';
+ const client = new FirecrawlClient({ apiKey });
+
+ const doc = await client.scrape('https://docs.firecrawl.dev', { formats: ['markdown'] });
+ console.log('scrape:', !!doc.markdown);
+
+ const crawl = await client.crawl('https://docs.firecrawl.dev', { limit: 3, pollInterval: 1, timeout: 120 });
+ console.log('crawl:', crawl.status, crawl.completed, '/', crawl.total);
+
+ const batch = await client.batchScrape([
+ 'https://docs.firecrawl.dev',
+ 'https://firecrawl.dev',
+ ], { options: { formats: ['markdown'] }, pollInterval: 1, timeout: 120 });
+ console.log('batch:', batch.status, batch.completed, '/', batch.total);
+
+ const search = await client.search('What is the capital of France?', { limit: 5 });
+ console.log('search web results:', (search.web || []).length);
+
+ const map = await client.map('https://firecrawl.dev');
+ console.log('map links:', map.links.length);
+}
+
+run().catch((e) => {
+ console.error(e);
+ if (globalThis.process && globalThis.process.exit) globalThis.process.exit(1);
+});
+
+// old stuff:
+
import FirecrawlApp from 'firecrawl';
import { z } from 'zod';
diff --git a/apps/js-sdk/example.ts b/apps/js-sdk/example.ts
index a8fff30a2..815c0c086 100644
--- a/apps/js-sdk/example.ts
+++ b/apps/js-sdk/example.ts
@@ -1,3 +1,41 @@
+// Placeholder v2 example (TypeScript)
+// Minimal usage of new FirecrawlClient. Replace with your API key before running.
+
+// import Firecrawl from '@mendable/firecrawl-js';
+import Firecrawl from './firecrawl/src/index';
+
+const run = async () => {
+ const apiKey = (globalThis as any).process?.env?.FIRECRAWL_API_KEY || 'fc-YOUR_API_KEY';
+ const apiUrl = (globalThis as any).process?.env?.FIRECRAWL_API_URL || 'https://api.firecrawl.dev';
+ const client = new Firecrawl({ apiKey, apiUrl });
+
+ const doc = await client.scrape('https://docs.firecrawl.dev', { formats: ['markdown'] });
+ console.log('scrape:', !!doc.markdown);
+
+ const crawl = await client.crawl('https://docs.firecrawl.dev', { limit: 3, pollInterval: 1, timeout: 120 });
+ console.log('crawl:', crawl.status, crawl.completed, '/', crawl.total);
+
+ const batch = await client.batchScrape([
+ 'https://docs.firecrawl.dev',
+ 'https://firecrawl.dev',
+ ], { options: { formats: ['markdown'] }, pollInterval: 1, timeout: 120 });
+ console.log('batch:', batch.status, batch.completed, '/', batch.total);
+
+ const search = await client.search('What is the capital of France?', { limit: 5 });
+ console.log('search web results:', (search.web || []).length);
+
+ const map = await client.map('https://firecrawl.dev');
+ console.log('map links:', map.links.length);
+};
+
+run().catch((e) => {
+ // eslint-disable-next-line no-console
+ console.error(e);
+ (globalThis as any).process?.exit?.(1);
+});
+
+// old stuff:
+
import FirecrawlApp, { CrawlStatusResponse, ErrorResponse } from 'firecrawl';
const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});
diff --git a/apps/js-sdk/example_v1.js b/apps/js-sdk/example_v1.js
new file mode 100644
index 000000000..7700e68af
--- /dev/null
+++ b/apps/js-sdk/example_v1.js
@@ -0,0 +1,19 @@
+import FirecrawlApp from 'firecrawl';
+
+// Placeholder v1 example (JavaScript)
+// Mirrors the older SDK usage. Replace with your API key before running.
+
+async function main() {
+ const app = new Firecrawl({ apiKey: process.env.FIRECRAWL_API_KEY || 'fc-YOUR_API_KEY' });
+
+ const scrape = await app.v1.scrapeUrl('firecrawl.dev');
+ if (scrape && scrape.success) console.log(scrape.markdown);
+
+ const crawl = await app.v1.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 3 });
+ console.log(crawl);
+}
+
+main().catch((e) => {
+ console.error(e);
+ process.exit(1);
+});
\ No newline at end of file
diff --git a/apps/js-sdk/example_v1.ts b/apps/js-sdk/example_v1.ts
new file mode 100644
index 000000000..d845a73ef
--- /dev/null
+++ b/apps/js-sdk/example_v1.ts
@@ -0,0 +1,23 @@
+// Placeholder v1 example (TypeScript)
+// Mirrors the older SDK usage. Replace with your API key before running.
+
+// import FirecrawlApp from 'firecrawl';
+import Firecrawl from './firecrawl/src/index'
+
+async function main() {
+ const app = new Firecrawl({ apiKey: process.env.FIRECRAWL_API_KEY || 'fc-YOUR_API_KEY' });
+
+ // Scrape a website (v1 style):
+ const scrape = await app.v1.scrapeUrl('firecrawl.dev');
+ if ((scrape as any).success) console.log((scrape as any).markdown);
+
+ // Crawl a website (v1 style):
+ const crawl = await app.v1.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 3 });
+ console.log(crawl);
+}
+
+main().catch((e) => {
+ console.error(e);
+ process.exit(1);
+});
+
diff --git a/apps/js-sdk/example_watcher.ts b/apps/js-sdk/example_watcher.ts
new file mode 100644
index 000000000..b163b82b5
--- /dev/null
+++ b/apps/js-sdk/example_watcher.ts
@@ -0,0 +1,40 @@
+/*
+ Advanced watcher example using the v2 FirecrawlClient.
+
+ Run with:
+ node --env-file=.env -r esbuild-register apps/js-sdk/firecrawl/src/examples/watcher.ts
+ or compile with your bundler, ensuring FIRECRAWL_API_KEY is set.
+*/
+
+import { FirecrawlClient } from "./firecrawl/src/v2/client";
+
+async function main() {
+ const apiKey = process.env.FIRECRAWL_API_KEY || "fc-YOUR_API_KEY";
+ const client = new FirecrawlClient({ apiKey });
+
+ // Start a crawl and attach a watcher for real-time updates
+ const start = await client.startCrawl("https://example.com", { limit: 5 });
+ console.log("Started crawl:", start.id);
+
+ const watcher = client.watcher(start.id, { kind: "crawl", pollInterval: 2, timeout: 120 });
+
+ watcher.on("document", (doc) => {
+ console.log("document:", (doc as any).url || (doc as any).metadata?.sourceURL || "<no url>");
+ });
+
+ watcher.on("snapshot", (snap) => {
+ console.log(`status: ${snap.status} (${snap.completed}/${snap.total})`);
+ });
+
+ watcher.on("done", (finalState) => {
+ console.log("done:", finalState.status, "docs:", finalState.data?.length ?? 0);
+ });
+
+ await watcher.start();
+}
+
+main().catch((e) => {
+ console.error(e);
+ process.exit(1);
+});
+
diff --git a/apps/js-sdk/firecrawl/.env.example b/apps/js-sdk/firecrawl/.env.example
index 6b1780bb8..81394d178 100644
--- a/apps/js-sdk/firecrawl/.env.example
+++ b/apps/js-sdk/firecrawl/.env.example
@@ -1,3 +1,2 @@
-API_URL=http://localhost:3002
-TEST_API_KEY=fc-YOUR_API_KEY
-
+FIRECRAWL_API_KEY=<your api key>
+FIRECRAWL_API_URL=https://api.firecrawl.dev
\ No newline at end of file
diff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.json
index 76a7d087d..02897e3fa 100644
--- a/apps/js-sdk/firecrawl/package-lock.json
+++ b/apps/js-sdk/firecrawl/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@mendable/firecrawl-js",
- "version": "1.29.1",
+ "version": "1.29.3",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@mendable/firecrawl-js",
- "version": "1.29.1",
+ "version": "1.29.3",
"license": "MIT",
"dependencies": {
"axios": "^1.11.0",
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 1aa0a0420..3b38b1fd8 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "1.29.3",
+ "version": "3.0.1",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
@@ -16,8 +16,8 @@
"build": "tsup",
"build-and-publish": "npm run build && npm publish --access public",
"publish-beta": "npm run build && npm publish --access public --tag beta",
- "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",
- "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"
+ "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/**/*.test.ts",
+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"
},
"repository": {
"type": "git",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v1/index.test.ts
similarity index 100%
rename from apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
rename to apps/js-sdk/firecrawl/src/__tests__/e2e/v1/index.test.ts
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.ts
new file mode 100644
index 000000000..3b3752896
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.ts
@@ -0,0 +1,122 @@
+/**
+ * E2E tests for v2 scrape
+ */
+import Firecrawl from "../../../../index";
+import { z } from "zod";
+import { config } from "dotenv";
+import { describe, test, expect } from "@jest/globals";
+
+config();
+
+const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";
+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
+
+describe("v2.scrape e2e", () => {
+ if (!API_KEY) {
+ console.warn("Skipping v2.scrape e2e: FIRECRAWL_API_KEY not set");
+ }
+
+ const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL })
+
+ const assertValidDocument = (doc: any) => {
+ expect(doc).toBeTruthy();
+ const hasContent = Boolean(doc.markdown?.length) || Boolean(doc.html?.length) || Boolean(doc.rawHtml?.length);
+ expect(hasContent).toBe(true);
+ expect(doc.metadata).toBeTruthy();
+ };
+
+ test("minimal: scrape only required params", async () => {
+ if (!client) throw new Error();
+ const doc = await client.scrape("https://docs.firecrawl.dev");
+ assertValidDocument(doc);
+ }, 60_000);
+
+ test("maximal: scrape with all options", async () => {
+ if (!client) throw new Error();
+ const doc = await client.scrape("https://docs.firecrawl.dev", {
+ formats: [
+ "markdown",
+ "html",
+ "rawHtml",
+ "links",
+ { type: "screenshot", fullPage: true, quality: 80, viewport: { width: 1280, height: 800 } },
+ {
+ type: "json",
+ prompt: "Summarize the page and list links",
+ schema: {
+ type: "object",
+ properties: {
+ summary: { type: "string" },
+ links: { type: "array", items: { type: "string", format: "uri" } },
+ },
+ required: ["summary"],
+ },
+ },
+ ],
+ parsers: ["pdf"],
+ headers: { "User-Agent": "firecrawl-tests" },
+ includeTags: ["article"],
+ excludeTags: ["nav"],
+ onlyMainContent: true,
+ waitFor: 1000,
+ timeout: 30_000,
+ location: { country: "us", languages: ["en"] },
+ mobile: false,
+ skipTlsVerification: false,
+ removeBase64Images: true,
+ blockAds: true,
+ proxy: "auto",
+ storeInCache: true,
+ maxAge: 60_000,
+ });
+ assertValidDocument(doc);
+ }, 90_000);
+
+ test("json format with zod schema (auto-converted internally)", async () => {
+ if (!client) throw new Error();
+ const zodSchema = z.object({
+ title: z.string().min(1),
+ items: z.array(z.string().url()).optional(),
+ });
+
+ const doc = await client.scrape("https://docs.firecrawl.dev", {
+ formats: [
+ {
+ type: "json",
+ prompt: "Extract title and items",
+ schema: zodSchema,
+ },
+ ],
+ });
+ expect(doc).toBeTruthy();
+ }, 90_000);
+
+ test.each([
+ ["markdown", "markdown"],
+ ["html", "html"],
+ ["rawHtml", "rawHtml"],
+ ["links", "links"],
+ ["screenshot", "screenshot"],
+ ])("basic format: %s", async (fmt, expectField) => {
+ if (!client) throw new Error();
+ const doc = await client.scrape("https://docs.firecrawl.dev", { formats: [fmt as any] });
+ if (expectField !== "links" && expectField !== "screenshot") {
+ assertValidDocument(doc);
+ }
+ if (expectField === "markdown") expect(doc.markdown).toBeTruthy();
+ if (expectField === "html") expect(doc.html).toBeTruthy();
+ if (expectField === "rawHtml") expect(doc.rawHtml).toBeTruthy();
+ if (expectField === "screenshot") expect(doc.screenshot).toBeTruthy();
+ if (expectField === "links") {
+ expect(Array.isArray(doc.links)).toBe(true);
+ expect((doc.links || []).length).toBeGreaterThan(0);
+ }
+ }, 90_000);
+
+ test("invalid url should throw", async () => {
+ if (!client) throw new Error();
+ await expect(client.scrape("")).rejects.toThrow("URL cannot be empty");
+ await expect(client.scrape(" ")).rejects.toThrow("URL cannot be empty");
+ });
+});
+
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.ts
new file mode 100644
index 000000000..80871eea5
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.ts
@@ -0,0 +1,210 @@
+/**
+ * E2E tests for v2 search (translated from Python tests)
+ */
+import Firecrawl from "../../../../index";
+import { config } from "dotenv";
+import { describe, test, expect } from "@jest/globals";
+
+config();
+
+const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";
+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
+
+const client = API_KEY ? new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL }) : null as any;
+
+function collectTexts(entries: any[] | undefined): string[] {
+ const texts: string[] = [];
+ for (const r of entries || []) {
+ const title = (r && typeof r === 'object') ? (r.title as unknown as string | undefined) : undefined;
+ const desc = (r && typeof r === 'object') ? (r.description as unknown as string | undefined) : undefined;
+ if (title) texts.push(String(title).toLowerCase());
+ if (desc) texts.push(String(desc).toLowerCase());
+ }
+ return texts;
+}
+
+function isDocument(entry: any): boolean {
+ return !!entry && (typeof entry.markdown === 'string' || typeof entry.html === 'string');
+}
+
+describe("v2.search e2e", () => {
+ if (!client) {
+ console.warn("Skipping v2.search e2e: FIRECRAWL_API_KEY not set");
+ }
+
+ test("minimal request", async () => {
+ if (!client) throw new Error();
+ const results = await client.search("What is the capital of France?");
+ expect(results).toBeTruthy();
+ expect(results).toHaveProperty("web");
+ expect(results).not.toHaveProperty("news");
+ expect(results).not.toHaveProperty("images");
+
+ expect(results.web).toBeTruthy();
+ expect((results.web || []).length).toBeGreaterThan(0);
+
+ for (const result of results.web || []) {
+ if (isDocument(result)) {
+ // documents appear if scraping happens
+ continue;
+ }
+ expect(typeof result.url).toBe("string");
+ expect(result.url.startsWith("http")).toBe(true);
+ expect(typeof result.title === "string" || result.title == null).toBe(true);
+ expect(typeof result.description === "string" || result.description == null).toBe(true);
+ }
+
+ const allText = collectTexts(results.web).join(" ");
+ expect(allText.includes("paris")).toBe(true);
+
+ expect(results.news == null).toBe(true);
+ expect(results.images == null).toBe(true);
+ }, 90_000);
+
+ test("with sources web+news and limit", async () => {
+ if (!client) throw new Error();
+ const results = await client.search("firecrawl", { sources: ["web", "news"], limit: 3 });
+ expect(results).toBeTruthy();
+ expect(results.web).toBeTruthy();
+ expect((results.web || []).length).toBeLessThanOrEqual(3);
+ if (results.news != null) {
+ expect((results.news || []).length).toBeLessThanOrEqual(3);
+ }
+ expect(results.images == null).toBe(true);
+
+ const webTitles = (results.web || []).map(r => (r?.title || "").toString().toLowerCase());
+ const webDescriptions = (results.web || []).map(r => (r?.description || "").toString().toLowerCase());
+ const allWebText = (webTitles.concat(webDescriptions)).join(" ");
+ expect(allWebText.includes("firecrawl")).toBe(true);
+ }, 90_000);
+
+ test("result structure", async () => {
+ if (!client) throw new Error();
+ const results = await client.search("test query", { limit: 1 });
+ if (results.web && results.web.length > 0) {
+ const result: any = results.web[0];
+ expect(result).toHaveProperty("url");
+ expect(result).toHaveProperty("title");
+ expect(result).toHaveProperty("description");
+ expect(typeof result.url).toBe("string");
+ expect(typeof result.title === "string" || result.title == null).toBe(true);
+ expect(typeof result.description === "string" || result.description == null).toBe(true);
+ expect(result.url.startsWith("http")).toBe(true);
+ }
+ }, 90_000);
+
+ test("all parameters (comprehensive)", async () => {
+ if (!client) throw new Error();
+ const schema = {
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ description: { type: "string" },
+ url: { type: "string" },
+ },
+ required: ["title", "description"],
+ } as const;
+
+ const results = await client.search("artificial intelligence", {
+ sources: [ "web", "news", "images" ],
+ limit: 3,
+ tbs: "qdr:m",
+ location: "US",
+ ignoreInvalidURLs: true,
+ timeout: 60_000,
+ scrapeOptions: {
+ formats: [
+ "markdown",
+ "html",
+ { type: "json", prompt: "Extract the title and description from the page", schema },
+ ],
+ headers: { "User-Agent": "Firecrawl-Test/1.0" },
+ includeTags: ["h1", "h2", "p"],
+ excludeTags: ["nav", "footer"],
+ onlyMainContent: true,
+ waitFor: 2000,
+ mobile: false,
+ skipTlsVerification: false,
+ removeBase64Images: true,
+ blockAds: true,
+ proxy: "basic",
+ maxAge: 3_600_000,
+ storeInCache: true,
+ location: { country: "US", languages: ["en"] },
+ actions: [{ type: "wait", milliseconds: 1000 }],
+ },
+ });
+
+ expect(results).toBeTruthy();
+ expect(results).toHaveProperty("web");
+ expect(results).toHaveProperty("news");
+ expect(results).toHaveProperty("images");
+
+ expect(results.web).toBeTruthy();
+ expect((results.web || []).length).toBeLessThanOrEqual(3);
+
+ const nonDocEntries = (results.web || []).filter(r => !isDocument(r));
+ if (nonDocEntries.length > 0) {
+ const allWebText = collectTexts(nonDocEntries).join(" ");
+ const aiTerms = ["artificial", "intelligence", "ai", "machine", "learning"];
+ expect(aiTerms.some(t => allWebText.includes(t))).toBe(true);
+ }
+
+ for (const result of results.web || []) {
+ if (isDocument(result)) {
+ expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true);
+ } else {
+ expect(typeof result.url).toBe("string");
+ expect(result.url.startsWith("http")).toBe(true);
+ }
+ }
+
+ if (results.news != null) {
+ expect((results.news || []).length).toBeLessThanOrEqual(3);
+ for (const result of results.news || []) {
+ if (isDocument(result)) {
+ expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true);
+ } else {
+ expect(typeof result.url).toBe("string");
+ expect(result.url.startsWith("http")).toBe(true);
+ }
+ }
+ }
+
+ expect(results.images == null).toBe(true);
+ }, 120_000);
+
+ test("formats flexibility: list vs object", async () => {
+ if (!client) throw new Error();
+ const results1 = await client.search("python programming", {
+ limit: 1,
+ scrapeOptions: { formats: ["markdown"] },
+ });
+ const results2 = await client.search("python programming", {
+ limit: 1,
+ scrapeOptions: { formats: [{ type: "markdown" as any }] },
+ });
+ expect(results1).toBeTruthy();
+ expect(results2).toBeTruthy();
+ expect(results1.web).toBeTruthy();
+ expect(results2.web).toBeTruthy();
+ }, 90_000);
+
+ test("with json format object", async () => {
+ if (!client) throw new Error();
+ const jsonSchema = {
+ type: "object",
+ properties: { title: { type: "string" } },
+ required: ["title"],
+ } as const;
+ const results = await client.search("site:docs.firecrawl.dev", {
+ limit: 1,
+ scrapeOptions: {
+ formats: [{ type: "json", prompt: "Extract page title", schema: jsonSchema }],
+ },
+ });
+ expect(results).toBeTruthy();
+ expect(Array.isArray(results.web) || results.web == null).toBe(true);
+ }, 90_000);
+});
+
diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/monitor-job-status-retry.test.ts
similarity index 100%
rename from apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts
rename to apps/js-sdk/firecrawl/src/__tests__/unit/v1/monitor-job-status-retry.test.ts
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape.unit.test.ts
new file mode 100644
index 000000000..1784cdc85
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape.unit.test.ts
@@ -0,0 +1,11 @@
+/**
+ * Minimal unit test for v2 scrape (no mocking; sanity check payload path)
+ */
+import { FirecrawlClient } from "../../../v2/client";
+
+describe("v2.scrape unit", () => {
+ test("constructor requires apiKey", () => {
+ expect(() => new FirecrawlClient({ apiKey: "", apiUrl: "https://api.firecrawl.dev" })).toThrow();
+ });
+});
+
diff --git a/apps/js-sdk/firecrawl/src/index.backup.ts b/apps/js-sdk/firecrawl/src/index.backup.ts
new file mode 100644
index 000000000..f93bb48e3
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/index.backup.ts
@@ -0,0 +1,2146 @@
+import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios";
+import * as zt from "zod";
+import { zodToJsonSchema } from "zod-to-json-schema";
+import { TypedEventTarget } from "typescript-event-target";
+
+/**
+ * Configuration interface for FirecrawlApp.
+ * @param apiKey - Optional API key for authentication.
+ * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
+ */
+export interface FirecrawlAppConfig {
+ apiKey?: string | null;
+ apiUrl?: string | null;
+}
+
+/**
+ * Metadata for a Firecrawl document.
+ * Includes various optional properties for document metadata.
+ */
+export interface FirecrawlDocumentMetadata {
+ title?: string;
+ description?: string;
+ language?: string;
+ keywords?: string;
+ robots?: string;
+ ogTitle?: string;
+ ogDescription?: string;
+ ogUrl?: string;
+ ogImage?: string;
+ ogAudio?: string;
+ ogDeterminer?: string;
+ ogLocale?: string;
+ ogLocaleAlternate?: string[];
+ ogSiteName?: string;
+ ogVideo?: string;
+ dctermsCreated?: string;
+ dcDateCreated?: string;
+ dcDate?: string;
+ dctermsType?: string;
+ dcType?: string;
+ dctermsAudience?: string;
+ dctermsSubject?: string;
+ dcSubject?: string;
+ dcDescription?: string;
+ dctermsKeywords?: string;
+ modifiedTime?: string;
+ publishedTime?: string;
+ articleTag?: string;
+ articleSection?: string;
+ sourceURL?: string;
+ statusCode?: number;
+ error?: string;
+ proxyUsed?: "basic" | "stealth";
+ cacheState?: "miss" | "hit";
+ cachedAt?: string;
+ [key: string]: any; // Allows for additional metadata properties not explicitly defined.
+}
+
+/**
+ * Document interface for Firecrawl.
+ * Represents a document retrieved or processed by Firecrawl.
+ */
+export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | never) = never> {
+ url?: string;
+ markdown?: string;
+ html?: string;
+ rawHtml?: string;
+ links?: string[];
+ extract?: T;
+ json?: T;
+ screenshot?: string;
+ metadata?: FirecrawlDocumentMetadata;
+ actions: ActionsSchema;
+ changeTracking?: {
+ previousScrapeAt: string | null;
+ changeStatus: "new" | "same" | "changed" | "removed";
+ visibility: "visible" | "hidden";
+ diff?: {
+ text: string;
+ json: {
+ files: Array<{
+ from: string | null;
+ to: string | null;
+ chunks: Array<{
+ content: string;
+ changes: Array<{
+ type: string;
+ normal?: boolean;
+ ln?: number;
+ ln1?: number;
+ ln2?: number;
+ content: string;
+ }>;
+ }>;
+ }>;
+ };
+ };
+ json?: any;
+ };
+ // v1 search only
+ title?: string;
+ description?: string;
+}
+
+/**
+ * Parameters for scraping operations.
+ * Defines the options and configurations available for scraping web content.
+ */
+export interface CrawlScrapeOptions {
+ formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "changeTracking")[];
+ headers?: Record<string, string>;
+ includeTags?: string[];
+ excludeTags?: string[];
+ onlyMainContent?: boolean;
+ waitFor?: number;
+ timeout?: number;
+ location?: {
+ country?: string;
+ languages?: string[];
+ };
+ mobile?: boolean;
+ skipTlsVerification?: boolean;
+ removeBase64Images?: boolean;
+ blockAds?: boolean;
+ proxy?: "basic" | "stealth" | "auto";
+ storeInCache?: boolean;
+ maxAge?: number;
+ parsePDF?: boolean;
+}
+
+export type Action = {
+ type: "wait",
+ milliseconds?: number,
+ selector?: string,
+} | {
+ type: "click",
+ selector: string,
+ all?: boolean,
+} | {
+ type: "screenshot",
+ fullPage?: boolean,
+ quality?: number,
+} | {
+ type: "write",
+ text: string,
+} | {
+ type: "press",
+ key: string,
+} | {
+ type: "scroll",
+ direction?: "up" | "down",
+ selector?: string,
+} | {
+ type: "scrape",
+} | {
+ type: "executeJavascript",
+ script: string,
+};
+
+export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema extends (Action[] | undefined) = undefined> extends CrawlScrapeOptions {
+ extract?: {
+ prompt?: string;
+ schema?: LLMSchema;
+ systemPrompt?: string;
+ };
+ jsonOptions?:{
+ prompt?: string;
+ schema?: LLMSchema;
+ systemPrompt?: string;
+ }
+ changeTrackingOptions?: {
+ prompt?: string;
+ schema?: any;
+ modes?: ("json" | "git-diff")[];
+ tag?: string | null;
+ }
+ actions?: ActionsSchema;
+ agent?: AgentOptions;
+ zeroDataRetention?: boolean;
+}
+
+export interface ActionsResult {
+ screenshots: string[];
+ scrapes: ({
+ url: string;
+ html: string;
+ })[];
+ javascriptReturns: {
+ type: string;
+ value: unknown
+ }[];
+}
+
+/**
+ * Response interface for scraping operations.
+ * Defines the structure of the response received after a scraping operation.
+ */
+export interface ScrapeResponse<LLMResult = any, ActionsSchema extends (ActionsResult | never) = never> extends FirecrawlDocument<LLMResult, ActionsSchema> {
+ success: true;
+ warning?: string;
+ error?: string;
+}
+
+/**
+ * Parameters for crawling operations.
+ * Includes options for both scraping and mapping during a crawl.
+ */
+export interface CrawlParams {
+ includePaths?: string[];
+ excludePaths?: string[];
+ maxDepth?: number;
+ maxDiscoveryDepth?: number;
+ limit?: number;
+ allowBackwardLinks?: boolean;
+ crawlEntireDomain?: boolean;
+ allowExternalLinks?: boolean;
+ ignoreSitemap?: boolean;
+ scrapeOptions?: CrawlScrapeOptions;
+ webhook?: string | {
+ url: string;
+ headers?: Record<string, string>;
+ metadata?: Record<string, string>;
+ events?: ["completed", "failed", "page", "started"][number][];
+ };
+ deduplicateSimilarURLs?: boolean;
+ ignoreQueryParameters?: boolean;
+ regexOnFullURL?: boolean;
+ /**
+ * Delay in seconds between scrapes. This helps respect website rate limits.
+ * If not provided, the crawler may use the robots.txt crawl delay if available.
+ */
+ delay?: number;
+ allowSubdomains?: boolean;
+ maxConcurrency?: number;
+ zeroDataRetention?: boolean;
+}
+
+/**
+ * Response interface for crawling operations.
+ * Defines the structure of the response received after initiating a crawl.
+ */
+export interface CrawlResponse {
+ id?: string;
+ url?: string;
+ success: true;
+ error?: string;
+}
+
+/**
+ * Response interface for batch scrape operations.
+ * Defines the structure of the response received after initiating a crawl.
+ */
+export interface BatchScrapeResponse {
+ id?: string;
+ url?: string;
+ success: true;
+ error?: string;
+ invalidURLs?: string[];
+}
+
+/**
+ * Response interface for job status checks.
+ * Provides detailed status of a crawl job including progress and results.
+ */
+export interface CrawlStatusResponse {
+ success: true;
+ status: "scraping" | "completed" | "failed" | "cancelled";
+ completed: number;
+ total: number;
+ creditsUsed: number;
+ expiresAt: Date;
+ next?: string;
+ data: FirecrawlDocument<undefined>[];
+};
+
+/**
+ * Response interface for batch scrape job status checks.
+ * Provides detailed status of a batch scrape job including progress and results.
+ */
+export interface BatchScrapeStatusResponse {
+ success: true;
+ status: "scraping" | "completed" | "failed" | "cancelled";
+ completed: number;
+ total: number;
+ creditsUsed: number;
+ expiresAt: Date;
+ next?: string;
+ data: FirecrawlDocument<undefined>[];
+};
+
+/**
+ * Parameters for mapping operations.
+ * Defines options for mapping URLs during a crawl.
+ */
+export interface MapParams {
+ search?: string;
+ ignoreSitemap?: boolean;
+ includeSubdomains?: boolean;
+ sitemapOnly?: boolean;
+ limit?: number;
+ timeout?: number;
+ useIndex?: boolean;
+}
+
+/**
+ * Response interface for mapping operations.
+ * Defines the structure of the response received after a mapping operation.
+ */
+export interface MapResponse {
+ success: true;
+ links?: string[];
+ error?: string;
+}
+
+/**
+ * Parameters for extracting information from URLs.
+ * Defines options for extracting information from URLs.
+ */
+export interface AgentOptions {
+ model?: string;
+ prompt?: string;
+ sessionId?: string;
+}
+
+/**
+ * Parameters for extracting information from URLs.
+ * Defines options for extracting information from URLs.
+ */
+export interface AgentOptionsExtract {
+ model?: string;
+ sessionId?: string;
+}
+
+export interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
+ prompt?: string;
+ schema?: LLMSchema | object;
+ systemPrompt?: string;
+ allowExternalLinks?: boolean;
+ enableWebSearch?: boolean;
+ includeSubdomains?: boolean;
+ origin?: string;
+ showSources?: boolean;
+ scrapeOptions?: CrawlScrapeOptions;
+ agent?: AgentOptionsExtract;
+}
+
+/**
+ * Response interface for extracting information from URLs.
+ * Defines the structure of the response received after extracting information from URLs.
+ */
+export interface ExtractResponse<LLMSchema extends zt.ZodSchema = any> {
+ success: boolean;
+ data: LLMSchema;
+ error?: string;
+ warning?: string;
+ sources?: string[];
+}
+
+/**
+ * Error response interface.
+ * Defines the structure of the response received when an error occurs.
+ */
+export interface ErrorResponse {
+ success: false;
+ error: string;
+}
+
+/**
+ * Custom error class for Firecrawl.
+ * Extends the built-in Error class to include a status code.
+ */
+export class FirecrawlError extends Error {
+ statusCode: number;
+ details?: any;
+ constructor(message: string, statusCode: number, details?: any) {
+ super(message);
+ this.statusCode = statusCode;
+ this.details = details;
+ }
+}
+
+/**
+ * Parameters for search operations.
+ * Defines options for searching and scraping search results.
+ */
+export interface SearchParams {
+ limit?: number;
+ tbs?: string;
+ filter?: string;
+ lang?: string;
+ country?: string;
+ location?: string;
+ origin?: string;
+ timeout?: number;
+ scrapeOptions?: ScrapeParams;
+}
+
+/**
+ * Response interface for search operations.
+ * Defines the structure of the response received after a search operation.
+ */
+export interface SearchResponse {
+ success: boolean;
+ data: FirecrawlDocument<undefined>[];
+ warning?: string;
+ error?: string;
+}
+
+/**
+ * Response interface for crawl/batch scrape error monitoring.
+ */
+export interface CrawlErrorsResponse {
+ /**
+ * Scrapes that errored out + error details
+ */
+ errors: {
+ id: string,
+ timestamp?: string,
+ url: string,
+ error: string,
+ }[];
+
+ /**
+ * URLs blocked by robots.txt
+ */
+ robotsBlocked: string[];
+};
+
+/**
+ * Parameters for deep research operations.
+ * Defines options for conducting deep research on a query.
+ */
+export interface DeepResearchParams<LLMSchema extends zt.ZodSchema = any> {
+ /**
+ * Maximum depth of research iterations (1-10)
+ * @default 7
+ */
+ maxDepth?: number;
+ /**
+ * Time limit in seconds (30-300)
+ * @default 270
+ */
+ timeLimit?: number;
+ /**
+ * Maximum number of URLs to analyze (1-1000)
+ * @default 20
+ */
+ maxUrls?: number;
+ /**
+ * The prompt to use for the final analysis
+ */
+ analysisPrompt?: string;
+ /**
+ * The system prompt to use for the research agent
+ */
+ systemPrompt?: string;
+ /**
+ * The formats to use for the final analysis
+ */
+ formats?: ("markdown" | "json")[];
+ /**
+ * The JSON options to use for the final analysis
+ */
+ jsonOptions?:{
+ prompt?: string;
+ schema?: LLMSchema;
+ systemPrompt?: string;
+ };
+ /**
+ * Experimental flag for streaming steps
+ */
+ // __experimental_streamSteps?: boolean;
+}
+
+/**
+ * Response interface for deep research operations.
+ */
+export interface DeepResearchResponse {
+ success: boolean;
+ id: string;
+}
+
+/**
+ * Status response interface for deep research operations.
+ */
+export interface DeepResearchStatusResponse {
+ success: boolean;
+ data: {
+ finalAnalysis: string;
+ activities: Array<{
+ type: string;
+ status: string;
+ message: string;
+ timestamp: string;
+ depth: number;
+ }>;
+ sources: Array<{
+ url: string;
+ title: string;
+ description: string;
+ }>;
+ };
+ status: "processing" | "completed" | "failed";
+ error?: string;
+ expiresAt: string;
+ currentDepth: number;
+ maxDepth: number;
+ activities: Array<{
+ type: string;
+ status: string;
+ message: string;
+ timestamp: string;
+ depth: number;
+ }>;
+ sources: Array<{
+ url: string;
+ title: string;
+ description: string;
+ }>;
+ summaries: string[];
+}
+
+/**
+ * Parameters for LLMs.txt generation operations.
+ */
+export interface GenerateLLMsTextParams {
+ /**
+ * Maximum number of URLs to process (1-100)
+ * @default 10
+ */
+ maxUrls?: number;
+ /**
+ * Whether to show the full LLMs-full.txt in the response
+ * @default false
+ */
+ showFullText?: boolean;
+ /**
+ * Whether to use cached content if available
+ * @default true
+ */
+ cache?: boolean;
+ /**
+ * Experimental flag for streaming
+ */
+ __experimental_stream?: boolean;
+}
+
+/**
+ * Response interface for LLMs.txt generation operations.
+ */
+export interface GenerateLLMsTextResponse {
+ success: boolean;
+ id: string;
+}
+
+/**
+ * Status response interface for LLMs.txt generation operations.
+ */
+export interface GenerateLLMsTextStatusResponse {
+ success: boolean;
+ data: {
+ llmstxt: string;
+ llmsfulltxt?: string;
+ };
+ status: "processing" | "completed" | "failed";
+ error?: string;
+ expiresAt: string;
+}
+
+/**
+ * Main class for interacting with the Firecrawl API.
+ * Provides methods for scraping, searching, crawling, and mapping web content.
+ */
+export default class FirecrawlApp {
+ public apiKey: string;
+ public apiUrl: string;
+ public version: string = "1.25.1";
+
+ private isCloudService(url: string): boolean {
+ return url.includes('api.firecrawl.dev');
+ }
+
+ private async getVersion(): Promise<string> {
+ try {
+ const packageJson = await import('../package.json', { assert: { type: 'json' } });
+ return packageJson.default.version;
+ } catch (error) {
+ console.error("Error getting version:", error);
+ return "1.25.1";
+ }
+ }
+
+ private async init() {
+ this.version = await this.getVersion();
+ }
+
+ /**
+ * Initializes a new instance of the FirecrawlApp class.
+ * @param config - Configuration options for the FirecrawlApp instance.
+ */
+ constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
+ const baseUrl = apiUrl || "https://api.firecrawl.dev";
+
+ if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
+ throw new FirecrawlError("No API key provided", 401);
+ }
+
+ this.apiKey = apiKey || '';
+ this.apiUrl = baseUrl;
+ this.init();
+ }
+
+ /**
+ * Scrapes a URL using the Firecrawl API.
+ * @param url - The URL to scrape.
+ * @param params - Additional parameters for the scrape request.
+ * @returns The response from the scrape operation.
+ */
+ async scrapeUrl<T extends zt.ZodSchema, ActionsSchema extends (Action[] | undefined) = undefined>(
+ url: string,
+ params?: ScrapeParams<T, ActionsSchema>
+ ): Promise<ScrapeResponse<zt.infer<T>, ActionsSchema extends Action[] ? ActionsResult : never> | ErrorResponse> {
+ const headers: AxiosRequestHeaders = {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${this.apiKey}`,
+ } as AxiosRequestHeaders;
+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
+ if (jsonData?.extract?.schema) {
+ let schema = jsonData.extract.schema;
+
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+
+ }
+ jsonData = {
+ ...jsonData,
+ extract: {
+ ...jsonData.extract,
+ schema: schema,
+ },
+ };
+ }
+
+ if (jsonData?.jsonOptions?.schema) {
+ let schema = jsonData.jsonOptions.schema;
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+
+ }
+ jsonData = {
+ ...jsonData,
+ jsonOptions: {
+ ...jsonData.jsonOptions,
+ schema: schema,
+ },
+ };
+ }
+ try {
+ const response: AxiosResponse = await axios.post(
+ this.apiUrl + `/v1/scrape`,
+ jsonData,
+ { headers, timeout: params?.timeout !== undefined ? (params.timeout + 5000) : undefined },
+ );
+ if (response.status === 200) {
+ const responseData = response.data;
+ if (responseData.success) {
+ return {
+ success: true,
+ warning: responseData.warning,
+ error: responseData.error,
+ ...responseData.data
+ };
+ } else {
+ throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
+ }
+ } else {
+ this.handleError(response, "scrape URL");
+ }
+ } catch (error: any) {
+ this.handleError(error.response, "scrape URL");
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Searches using the Firecrawl API and optionally scrapes the results.
+ * @param query - The search query string.
+ * @param params - Optional parameters for the search request.
+ * @returns The response from the search operation.
+ */
+ async search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse> {
+ const headers: AxiosRequestHeaders = {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${this.apiKey}`,
+ } as AxiosRequestHeaders;
+
+ let jsonData: any = {
+ query,
+ limit: params?.limit ?? 5,
+ tbs: params?.tbs,
+ filter: params?.filter,
+ lang: params?.lang ?? "en",
+ country: params?.country ?? "us",
+ location: params?.location,
+ origin: `js-sdk@${this.version}`,
+ timeout: params?.timeout ?? 60000,
+ scrapeOptions: params?.scrapeOptions ?? { formats: [] },
+ };
+
+ if (jsonData?.scrapeOptions?.extract?.schema) {
+ let schema = jsonData.scrapeOptions.extract.schema;
+
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+
+ }
+ jsonData = {
+ ...jsonData,
+ scrapeOptions: {
+ ...jsonData.scrapeOptions,
+ extract: {
+ ...jsonData.scrapeOptions.extract,
+ schema: schema,
+ },
+ },
+ };
+ }
+
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/search`,
+ jsonData,
+ headers
+ );
+
+ if (response.status === 200) {
+ const responseData = response.data;
+ if (responseData.success) {
+ return {
+ success: true,
+ data: responseData.data as FirecrawlDocument<any>[],
+ warning: responseData.warning,
+ };
+ } else {
+ throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);
+ }
+ } else {
+ this.handleError(response, "search");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error.", data: [] };
+ }
+
+ /**
+ * Initiates a crawl job for a URL using the Firecrawl API.
+ * @param url - The URL to crawl.
+ * @param params - Additional parameters for the crawl request.
+ * @param pollInterval - Time in seconds for job status checks.
+ * @param idempotencyKey - Optional idempotency key for the request.
+ * @returns The response from the crawl operation.
+ */
+ async crawlUrl(
+ url: string,
+ params?: CrawlParams,
+ pollInterval: number = 2,
+ idempotencyKey?: string
+ ): Promise<CrawlStatusResponse | ErrorResponse> {
+ const headers = this.prepareHeaders(idempotencyKey);
+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/crawl`,
+ jsonData,
+ headers
+ );
+ if (response.status === 200) {
+ const id: string = response.data.id;
+ return this.monitorJobStatus(id, headers, pollInterval);
+ } else {
+ this.handleError(response, "start crawl job");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ async asyncCrawlUrl(
+ url: string,
+ params?: CrawlParams,
+ idempotencyKey?: string
+ ): Promise<CrawlResponse | ErrorResponse> {
+ const headers = this.prepareHeaders(idempotencyKey);
+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/crawl`,
+ jsonData,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start crawl job");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Checks the status of a crawl job using the Firecrawl API.
+ * @param id - The ID of the crawl operation.
+ * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+ * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+ * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+ * @param limit - How many entries to return. Only used when `getAllData = false`.
+ * @returns The response containing the job status.
+ */
+ async checkCrawlStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse> {
+ if (!id) {
+ throw new FirecrawlError("No crawl ID provided", 400);
+ }
+
+ const headers: AxiosRequestHeaders = this.prepareHeaders();
+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+ if (skip !== undefined) {
+ targetURL.searchParams.set("skip", skip.toString());
+ }
+ if (limit !== undefined) {
+ targetURL.searchParams.set("limit", limit.toString());
+ }
+
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ targetURL.href,
+ headers
+ );
+ if (response.status === 200) {
+ let allData = response.data.data;
+ if (getAllData && response.data.status === "completed") {
+ let statusData = response.data
+ if ("data" in statusData) {
+ let data = statusData.data;
+ while (typeof statusData === 'object' && 'next' in statusData) {
+ if (data.length === 0) {
+ break
+ }
+ statusData = (await this.getRequest(statusData.next, headers)).data;
+ data = data.concat(statusData.data);
+ }
+ allData = data;
+ }
+ }
+
+ let resp: CrawlStatusResponse | ErrorResponse = {
+ success: response.data.success,
+ status: response.data.status,
+ total: response.data.total,
+ completed: response.data.completed,
+ creditsUsed: response.data.creditsUsed,
+ next: getAllData ? undefined : response.data.next,
+ expiresAt: new Date(response.data.expiresAt),
+ data: allData
+ }
+
+ if (!response.data.success && response.data.error) {
+ resp = {
+ ...resp,
+ success: false,
+ error: response.data.error
+ } as ErrorResponse;
+ }
+
+ if (response.data.next) {
+ (resp as CrawlStatusResponse).next = response.data.next;
+ }
+
+ return resp;
+ } else {
+ this.handleError(response, "check crawl status");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Returns information about crawl errors.
+ * @param id - The ID of the crawl operation.
+ * @returns Information about crawl errors.
+ */
+ async checkCrawlErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.deleteRequest(
+ `${this.apiUrl}/v1/crawl/${id}/errors`,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "check crawl errors");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Cancels a crawl job using the Firecrawl API.
+ * @param id - The ID of the crawl operation.
+ * @returns The response from the cancel crawl operation.
+ */
+ async cancelCrawl(id: string): Promise<ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.deleteRequest(
+ `${this.apiUrl}/v1/crawl/${id}`,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "cancel crawl job");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Initiates a crawl job and returns a CrawlWatcher to monitor the job via WebSocket.
+ * @param url - The URL to crawl.
+ * @param params - Additional parameters for the crawl request.
+ * @param idempotencyKey - Optional idempotency key for the request.
+ * @returns A CrawlWatcher instance to monitor the crawl job.
+ */
+ async crawlUrlAndWatch(
+ url: string,
+ params?: CrawlParams,
+ idempotencyKey?: string,
+ ) {
+ const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
+
+ if (crawl.success && crawl.id) {
+ const id = crawl.id;
+ return new CrawlWatcher(id, this);
+ }
+
+ throw new FirecrawlError("Crawl job failed to start", 400);
+ }
+
+ /**
+ * Maps a URL using the Firecrawl API.
+ * @param url - The URL to map.
+ * @param params - Additional parameters for the map request.
+ * @returns The response from the map operation.
+ */
+ async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
+
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/map`,
+ jsonData,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data as MapResponse;
+ } else {
+ this.handleError(response, "map");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Initiates a batch scrape job for multiple URLs using the Firecrawl API.
+ * @param url - The URLs to scrape.
+ * @param params - Additional parameters for the scrape request.
+ * @param pollInterval - Time in seconds for job status checks.
+ * @param idempotencyKey - Optional idempotency key for the request.
+ * @param webhook - Optional webhook for the batch scrape.
+ * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
+ * @returns The response from the crawl operation.
+ */
+ async batchScrapeUrls(
+ urls: string[],
+ params?: ScrapeParams,
+ pollInterval: number = 2,
+ idempotencyKey?: string,
+ webhook?: CrawlParams["webhook"],
+ ignoreInvalidURLs?: boolean,
+ maxConcurrency?: number,
+ ): Promise<BatchScrapeStatusResponse | ErrorResponse> {
+ const headers = this.prepareHeaders(idempotencyKey);
+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: `js-sdk@${this.version}` };
+ if (jsonData?.extract?.schema) {
+ let schema = jsonData.extract.schema;
+
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+
+ }
+ jsonData = {
+ ...jsonData,
+ extract: {
+ ...jsonData.extract,
+ schema: schema,
+ },
+ };
+ }
+ if (jsonData?.jsonOptions?.schema) {
+ let schema = jsonData.jsonOptions.schema;
+
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+
+ }
+ jsonData = {
+ ...jsonData,
+ jsonOptions: {
+ ...jsonData.jsonOptions,
+ schema: schema,
+ },
+ };
+ }
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/batch/scrape`,
+ jsonData,
+ headers
+ );
+ if (response.status === 200) {
+ const id: string = response.data.id;
+ return this.monitorJobStatus(id, headers, pollInterval);
+ } else {
+ this.handleError(response, "start batch scrape job");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ async asyncBatchScrapeUrls(
+ urls: string[],
+ params?: ScrapeParams,
+ idempotencyKey?: string,
+ webhook?: CrawlParams["webhook"],
+ ignoreInvalidURLs?: boolean,
+ ): Promise<BatchScrapeResponse | ErrorResponse> {
+ const headers = this.prepareHeaders(idempotencyKey);
+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/batch/scrape`,
+ jsonData,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start batch scrape job");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.
+ * @param urls - The URL to scrape.
+ * @param params - Additional parameters for the scrape request.
+ * @param idempotencyKey - Optional idempotency key for the request.
+ * @returns A CrawlWatcher instance to monitor the crawl job.
+ */
+ async batchScrapeUrlsAndWatch(
+ urls: string[],
+ params?: ScrapeParams,
+ idempotencyKey?: string,
+ webhook?: CrawlParams["webhook"],
+ ignoreInvalidURLs?: boolean,
+ ) {
+ const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
+
+ if (crawl.success && crawl.id) {
+ const id = crawl.id;
+ return new CrawlWatcher(id, this);
+ }
+
+ throw new FirecrawlError("Batch scrape job failed to start", 400);
+ }
+
+ /**
+ * Checks the status of a batch scrape job using the Firecrawl API.
+ * @param id - The ID of the batch scrape operation.
+ * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+ * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+ * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+ * @param limit - How many entries to return. Only used when `getAllData = false`.
+ * @returns The response containing the job status.
+ */
+ async checkBatchScrapeStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse> {
+ if (!id) {
+ throw new FirecrawlError("No batch scrape ID provided", 400);
+ }
+
+ const headers: AxiosRequestHeaders = this.prepareHeaders();
+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+ if (skip !== undefined) {
+ targetURL.searchParams.set("skip", skip.toString());
+ }
+ if (limit !== undefined) {
+ targetURL.searchParams.set("limit", limit.toString());
+ }
+
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ targetURL.href,
+ headers
+ );
+ if (response.status === 200) {
+ let allData = response.data.data;
+ if (getAllData && response.data.status === "completed") {
+ let statusData = response.data
+ if ("data" in statusData) {
+ let data = statusData.data;
+ while (typeof statusData === 'object' && 'next' in statusData) {
+ if (data.length === 0) {
+ break
+ }
+ statusData = (await this.getRequest(statusData.next, headers)).data;
+ data = data.concat(statusData.data);
+ }
+ allData = data;
+ }
+ }
+
+ let resp: BatchScrapeStatusResponse | ErrorResponse = {
+ success: response.data.success,
+ status: response.data.status,
+ total: response.data.total,
+ completed: response.data.completed,
+ creditsUsed: response.data.creditsUsed,
+ next: getAllData ? undefined : response.data.next,
+ expiresAt: new Date(response.data.expiresAt),
+ data: allData
+ }
+
+ if (!response.data.success && response.data.error) {
+ resp = {
+ ...resp,
+ success: false,
+ error: response.data.error
+ } as ErrorResponse;
+ }
+
+ if (response.data.next) {
+ (resp as BatchScrapeStatusResponse).next = response.data.next;
+ }
+
+ return resp;
+ } else {
+ this.handleError(response, "check batch scrape status");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Returns information about batch scrape errors.
+ * @param id - The ID of the batch scrape operation.
+ * @returns Information about batch scrape errors.
+ */
+ async checkBatchScrapeErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.deleteRequest(
+ `${this.apiUrl}/v1/batch/scrape/${id}/errors`,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "check batch scrape errors");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Extracts information from URLs using the Firecrawl API.
+ * Currently in Beta. Expect breaking changes on future minor versions.
+ * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
+ * @param params - Additional parameters for the extract request.
+ * @returns The response from the extract operation.
+ */
+ async extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {
+ const headers = this.prepareHeaders();
+
+ let jsonData: { urls?: string[] } & ExtractParams<T> = { urls: urls, ...params };
+ let jsonSchema: any;
+ try {
+ if (!params?.schema) {
+ jsonSchema = undefined;
+ } else {
+ try {
+ jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);
+ } catch (_) {
+ jsonSchema = params.schema;
+ }
+ }
+ } catch (error: any) {
+ throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
+ }
+
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/extract`,
+ { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
+ headers
+ );
+
+ if (response.status === 200) {
+ const jobId = response.data.id;
+ let extractStatus;
+ do {
+ const statusResponse: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/extract/${jobId}`,
+ headers
+ );
+ extractStatus = statusResponse.data;
+ if (extractStatus.status === "completed") {
+ if (extractStatus.success) {
+ return {
+ success: true,
+ data: extractStatus.data,
+ warning: extractStatus.warning,
+ error: extractStatus.error,
+ sources: extractStatus?.sources || undefined,
+ };
+ } else {
+ throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);
+ }
+ } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {
+ throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);
+ }
+ await new Promise(resolve => setTimeout(resolve, 1000)); // Polling interval
+ } while (extractStatus.status !== "completed");
+ } else {
+ this.handleError(response, "extract");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);
+ }
+ return { success: false, error: "Internal server error."};
+ }
+
+ /**
+ * Initiates an asynchronous extract job for a URL using the Firecrawl API.
+ * @param url - The URL to extract data from.
+ * @param params - Additional parameters for the extract request.
+ * @param idempotencyKey - Optional idempotency key for the request.
+ * @returns The response from the extract operation.
+ */
+ async asyncExtract(
+ urls: string[],
+ params?: ExtractParams,
+ idempotencyKey?: string
+ ): Promise<ExtractResponse | ErrorResponse> {
+ const headers = this.prepareHeaders(idempotencyKey);
+ let jsonData: any = { urls, ...params };
+ let jsonSchema: any;
+
+ try {
+ if (!params?.schema) {
+ jsonSchema = undefined;
+ } else {
+ try {
+ jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);
+ } catch (_) {
+ jsonSchema = params.schema;
+ }
+ }
+ } catch (error: any) {
+ throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
+ }
+
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/extract`,
+ { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start extract job");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Retrieves the status of an extract job.
+ * @param jobId - The ID of the extract job.
+ * @returns The status of the extract job.
+ */
+ async getExtractStatus(jobId: string): Promise<any> {
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/extract/${jobId}`,
+ this.prepareHeaders()
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "get extract status");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+
+ /**
+ * Prepares the headers for an API request.
+ * @param idempotencyKey - Optional key to ensure idempotency.
+ * @returns The prepared headers.
+ */
+ prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
+ return {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${this.apiKey}`,
+ ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),
+ } as AxiosRequestHeaders & { "x-idempotency-key"?: string };
+ }
+
+ /**
+ * Sends a POST request to the specified URL.
+ * @param url - The URL to send the request to.
+ * @param data - The data to send in the request.
+ * @param headers - The headers for the request.
+ * @returns The response from the POST request.
+ */
+ postRequest(
+ url: string,
+ data: any,
+ headers: AxiosRequestHeaders
+ ): Promise<AxiosResponse> {
+ return axios.post(url, data, { headers, timeout: (data?.timeout ? (data.timeout + 5000) : undefined) });
+ }
+
+ /**
+ * Sends a GET request to the specified URL.
+ * @param url - The URL to send the request to.
+ * @param headers - The headers for the request.
+ * @returns The response from the GET request.
+ */
+ async getRequest(
+ url: string,
+ headers: AxiosRequestHeaders
+ ): Promise<AxiosResponse> {
+ try {
+ return await axios.get(url, { headers });
+ } catch (error) {
+ if (error instanceof AxiosError && error.response) {
+ return error.response as AxiosResponse;
+ } else {
+ throw error;
+ }
+ }
+ }
+
+ /**
+ * Sends a DELETE request to the specified URL.
+ * @param url - The URL to send the request to.
+ * @param headers - The headers for the request.
+ * @returns The response from the DELETE request.
+ */
+ async deleteRequest(
+ url: string,
+ headers: AxiosRequestHeaders
+ ): Promise<AxiosResponse> {
+ try {
+ return await axios.delete(url, { headers });
+ } catch (error) {
+ if (error instanceof AxiosError && error.response) {
+ return error.response as AxiosResponse;
+ } else {
+ throw error;
+ }
+ }
+ }
+
+ /**
+ * Monitors the status of a crawl job until completion or failure.
+ * @param id - The ID of the crawl operation.
+ * @param headers - The headers for the request.
+ * @param checkInterval - Interval in seconds for job status checks.
+ * @param checkUrl - Optional URL to check the status (used for v1 API)
+ * @returns The final job status or data.
+ */
+ async monitorJobStatus(
+ id: string,
+ headers: AxiosRequestHeaders,
+ checkInterval: number
+ ): Promise<CrawlStatusResponse | ErrorResponse> {
+ let failedTries = 0;
+ let networkRetries = 0;
+ const maxNetworkRetries = 3;
+
+ while (true) {
+ try {
+ let statusResponse: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/crawl/${id}`,
+ headers
+ );
+
+ if (statusResponse.status === 200) {
+ failedTries = 0;
+ networkRetries = 0;
+ let statusData = statusResponse.data;
+
+ if (statusData.status === "completed") {
+ if ("data" in statusData) {
+ let data = statusData.data;
+ while (typeof statusData === 'object' && 'next' in statusData) {
+ if (data.length === 0) {
+ break
+ }
+ statusResponse = await this.getRequest(statusData.next, headers);
+ statusData = statusResponse.data;
+ data = data.concat(statusData.data);
+ }
+ statusData.data = data;
+ return statusData;
+ } else {
+ throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+ }
+ } else if (
+ ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
+ ) {
+ checkInterval = Math.max(checkInterval, 2);
+ await new Promise((resolve) =>
+ setTimeout(resolve, checkInterval * 1000)
+ );
+ } else {
+ throw new FirecrawlError(
+ `Crawl job failed or was stopped. Status: ${statusData.status}`,
+ 500
+ );
+ }
+ } else {
+ failedTries++;
+ if (failedTries >= 3) {
+ this.handleError(statusResponse, "check crawl status");
+ }
+ }
+ } catch (error: any) {
+ if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {
+ networkRetries++;
+ const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);
+
+ await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+ continue;
+ }
+
+ throw new FirecrawlError(error, 500);
+ }
+ }
+ }
+
+ /**
+ * Determines if an error is retryable (transient network error)
+ * @param error - The error to check
+ * @returns True if the error should be retried
+ */
+ private isRetryableError(error: any): boolean {
+ if (error instanceof AxiosError) {
+ if (!error.response) {
+ const code = error.code;
+ const message = error.message?.toLowerCase() || '';
+
+ return (
+ code === 'ECONNRESET' ||
+ code === 'ETIMEDOUT' ||
+ code === 'ENOTFOUND' ||
+ code === 'ECONNREFUSED' ||
+ message.includes('socket hang up') ||
+ message.includes('network error') ||
+ message.includes('timeout')
+ );
+ }
+
+ if (error.response?.status === 408 || error.response?.status === 504) {
+ return true;
+ }
+ }
+
+ if (error && typeof error === 'object') {
+ const code = error.code;
+ const message = error.message?.toLowerCase() || '';
+
+ if (code === 'ECONNRESET' ||
+ code === 'ETIMEDOUT' ||
+ code === 'ENOTFOUND' ||
+ code === 'ECONNREFUSED' ||
+ message.includes('socket hang up') ||
+ message.includes('network error') ||
+ message.includes('timeout')) {
+ return true;
+ }
+
+ if (error.response?.status === 408 || error.response?.status === 504) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Handles errors from API responses.
+ * @param {AxiosResponse} response - The response from the API.
+ * @param {string} action - The action being performed when the error occurred.
+ */
+ handleError(response: AxiosResponse, action: string): void {
+ if (!response) {
+ throw new FirecrawlError(
+ `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,
+ 0
+ );
+ }
+
+ if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
+ const errorMessage: string =
+ response.data.error || "Unknown error occurred";
+ const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : '';
+ throw new FirecrawlError(
+ `Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}${details}`,
+ response.status,
+ response?.data?.details
+ );
+ } else {
+ throw new FirecrawlError(
+ `Unexpected error occurred while trying to ${action}. Status code: ${response.status}`,
+ response.status
+ );
+ }
+ }
+
+ /**
+ * Initiates a deep research operation on a given query and polls until completion.
+ * @param query - The query to research.
+ * @param params - Parameters for the deep research operation.
+ * @param onActivity - Optional callback to receive activity updates in real-time.
+ * @param onSource - Optional callback to receive source updates in real-time.
+ * @returns The final research results.
+ */
+ async deepResearch(
+ query: string,
+ params: DeepResearchParams<zt.ZodSchema>,
+ onActivity?: (activity: {
+ type: string;
+ status: string;
+ message: string;
+ timestamp: string;
+ depth: number;
+ }) => void,
+ onSource?: (source: {
+ url: string;
+ title?: string;
+ description?: string;
+ icon?: string;
+ }) => void
+ ): Promise<DeepResearchStatusResponse | ErrorResponse> {
+ try {
+ const response = await this.asyncDeepResearch(query, params);
+
+ if (!response.success || 'error' in response) {
+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };
+ }
+
+ if (!response.id) {
+ throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);
+ }
+
+ const jobId = response.id;
+ let researchStatus;
+ let lastActivityCount = 0;
+ let lastSourceCount = 0;
+
+ while (true) {
+ researchStatus = await this.checkDeepResearchStatus(jobId);
+
+ if ('error' in researchStatus && !researchStatus.success) {
+ return researchStatus;
+ }
+
+ // Stream new activities through the callback if provided
+ if (onActivity && researchStatus.activities) {
+ const newActivities = researchStatus.activities.slice(lastActivityCount);
+ for (const activity of newActivities) {
+ onActivity(activity);
+ }
+ lastActivityCount = researchStatus.activities.length;
+ }
+
+ // Stream new sources through the callback if provided
+ if (onSource && researchStatus.sources) {
+ const newSources = researchStatus.sources.slice(lastSourceCount);
+ for (const source of newSources) {
+ onSource(source);
+ }
+ lastSourceCount = researchStatus.sources.length;
+ }
+
+ if (researchStatus.status === "completed") {
+ return researchStatus;
+ }
+
+ if (researchStatus.status === "failed") {
+ throw new FirecrawlError(
+ `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,
+ 500
+ );
+ }
+
+ if (researchStatus.status !== "processing") {
+ break;
+ }
+
+ await new Promise(resolve => setTimeout(resolve, 2000));
+ }
+
+ return { success: false, error: "Research job terminated unexpectedly" };
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);
+ }
+ }
+
+ /**
+ * Initiates a deep research operation on a given query without polling.
+ * @param params - Parameters for the deep research operation.
+ * @returns The response containing the research job ID.
+ */
+ async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ let jsonData: any = { query, ...params, origin: `js-sdk@${this.version}` };
+
+ if (jsonData?.jsonOptions?.schema) {
+ let schema = jsonData.jsonOptions.schema;
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+ // Ignore error if schema can't be parsed as Zod
+ }
+ jsonData = {
+ ...jsonData,
+ jsonOptions: {
+ ...jsonData.jsonOptions,
+ schema: schema,
+ },
+ };
+ }
+
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ `${this.apiUrl}/v1/deep-research`,
+ jsonData,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start deep research");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Checks the status of a deep research operation.
+ * @param id - The ID of the deep research operation.
+ * @returns The current status and results of the research operation.
+ */
+ async checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/deep-research/${id}`,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else if (response.status === 404) {
+ throw new FirecrawlError("Deep research job not found", 404);
+ } else {
+ this.handleError(response, "check deep research status");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * @deprecated Use deepResearch() instead
+ * Initiates a deep research operation on a given topic and polls until completion.
+ * @param topic - The topic to research.
+ * @param params - Parameters for the deep research operation.
+ * @param onActivity - Optional callback to receive activity updates in real-time.
+ * @returns The final research results.
+ */
+ async __deepResearch(
+ topic: string,
+ params: DeepResearchParams,
+ onActivity?: (activity: {
+ type: string;
+ status: string;
+ message: string;
+ timestamp: string;
+ depth: number;
+ }) => void
+ ): Promise<DeepResearchStatusResponse | ErrorResponse> {
+ try {
+ const response = await this.__asyncDeepResearch(topic, params);
+
+ if (!response.success || 'error' in response) {
+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };
+ }
+
+ if (!response.id) {
+ throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);
+ }
+
+ const jobId = response.id;
+ let researchStatus;
+ let lastActivityCount = 0;
+
+ while (true) {
+ researchStatus = await this.__checkDeepResearchStatus(jobId);
+
+ if ('error' in researchStatus && !researchStatus.success) {
+ return researchStatus;
+ }
+
+ // Stream new activities through the callback if provided
+ if (onActivity && researchStatus.activities) {
+ const newActivities = researchStatus.activities.slice(lastActivityCount);
+ for (const activity of newActivities) {
+ onActivity(activity);
+ }
+ lastActivityCount = researchStatus.activities.length;
+ }
+
+ if (researchStatus.status === "completed") {
+ return researchStatus;
+ }
+
+ if (researchStatus.status === "failed") {
+ throw new FirecrawlError(
+ `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,
+ 500
+ );
+ }
+
+ if (researchStatus.status !== "processing") {
+ break;
+ }
+
+ await new Promise(resolve => setTimeout(resolve, 2000));
+ }
+
+ return { success: false, error: "Research job terminated unexpectedly" };
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);
+ }
+ }
+
+ /**
+ * @deprecated Use asyncDeepResearch() instead
+ * Initiates a deep research operation on a given topic without polling.
+ * @param params - Parameters for the deep research operation.
+ * @returns The response containing the research job ID.
+ */
+ async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ let jsonData: any = { topic, ...params, origin: `js-sdk@${this.version}` };
+ const response: AxiosResponse = await this.postRequest(
+ `${this.apiUrl}/v1/deep-research`,
+ jsonData,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start deep research");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * @deprecated Use checkDeepResearchStatus() instead
+ * Checks the status of a deep research operation.
+ * @param id - The ID of the deep research operation.
+ * @returns The current status and results of the research operation.
+ */
+ async __checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/deep-research/${id}`,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else if (response.status === 404) {
+ throw new FirecrawlError("Deep research job not found", 404);
+ } else {
+ this.handleError(response, "check deep research status");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Generates LLMs.txt for a given URL and polls until completion.
+ * @param url - The URL to generate LLMs.txt from.
+ * @param params - Parameters for the LLMs.txt generation operation.
+ * @returns The final generation results.
+ */
+ async generateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {
+ try {
+ const response = await this.asyncGenerateLLMsText(url, params);
+
+ if (!response.success || 'error' in response) {
+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };
+ }
+
+ if (!response.id) {
+ throw new FirecrawlError(`Failed to start LLMs.txt generation. No job ID returned.`, 500);
+ }
+
+ const jobId = response.id;
+ let generationStatus;
+
+ while (true) {
+ generationStatus = await this.checkGenerateLLMsTextStatus(jobId);
+
+ if ('error' in generationStatus && !generationStatus.success) {
+ return generationStatus;
+ }
+
+ if (generationStatus.status === "completed") {
+ return generationStatus;
+ }
+
+ if (generationStatus.status === "failed") {
+ throw new FirecrawlError(
+ `LLMs.txt generation job ${generationStatus.status}. Error: ${generationStatus.error}`,
+ 500
+ );
+ }
+
+ if (generationStatus.status !== "processing") {
+ break;
+ }
+
+ await new Promise(resolve => setTimeout(resolve, 2000));
+ }
+
+ return { success: false, error: "LLMs.txt generation job terminated unexpectedly" };
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);
+ }
+ }
+
+ /**
+ * Initiates a LLMs.txt generation operation without polling.
+ * @param url - The URL to generate LLMs.txt from.
+ * @param params - Parameters for the LLMs.txt generation operation.
+ * @returns The response containing the generation job ID.
+ */
+ async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ `${this.apiUrl}/v1/llmstxt`,
+ jsonData,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start LLMs.txt generation");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Checks the status of a LLMs.txt generation operation.
+ * @param id - The ID of the LLMs.txt generation operation.
+ * @returns The current status and results of the generation operation.
+ */
+ async checkGenerateLLMsTextStatus(id: string): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/llmstxt/${id}`,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else if (response.status === 404) {
+ throw new FirecrawlError("LLMs.txt generation job not found", 404);
+ } else {
+ this.handleError(response, "check LLMs.txt generation status");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+}
+
+interface CrawlWatcherEvents {
+ document: CustomEvent<FirecrawlDocument<undefined>>,
+ done: CustomEvent<{
+ status: CrawlStatusResponse["status"];
+ data: FirecrawlDocument<undefined>[];
+ }>,
+ error: CustomEvent<{
+ status: CrawlStatusResponse["status"],
+ data: FirecrawlDocument<undefined>[],
+ error: string,
+ }>,
+}
+
+export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
+ private ws: WebSocket;
+ public data: FirecrawlDocument<undefined>[];
+ public status: CrawlStatusResponse["status"];
+ public id: string;
+
+ constructor(id: string, app: FirecrawlApp) {
+ super();
+ this.id = id;
+ // replace `http` with `ws` (`http://` -> `ws://` and `https://` -> `wss://`)
+ const wsUrl = app.apiUrl.replace(/^http/, "ws");
+ this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
+ this.status = "scraping";
+ this.data = [];
+
+ type ErrorMessage = {
+ type: "error",
+ error: string,
+ }
+
+ type CatchupMessage = {
+ type: "catchup",
+ data: CrawlStatusResponse,
+ }
+
+ type DocumentMessage = {
+ type: "document",
+ data: FirecrawlDocument<undefined>,
+ }
+
+ type DoneMessage = { type: "done" }
+
+ type Message = ErrorMessage | CatchupMessage | DoneMessage | DocumentMessage;
+
+ const messageHandler = (msg: Message) => {
+ if (msg.type === "done") {
+ this.status = "completed";
+ this.dispatchTypedEvent("done", new CustomEvent("done", {
+ detail: {
+ status: this.status,
+ data: this.data,
+ id: this.id,
+ },
+ }));
+ } else if (msg.type === "error") {
+ this.status = "failed";
+ this.dispatchTypedEvent("error", new CustomEvent("error", {
+ detail: {
+ status: this.status,
+ data: this.data,
+ error: msg.error,
+ id: this.id,
+ },
+ }));
+ } else if (msg.type === "catchup") {
+ this.status = msg.data.status;
+ this.data.push(...(msg.data.data ?? []));
+ for (const doc of this.data) {
+ this.dispatchTypedEvent("document", new CustomEvent("document", {
+ detail: {
+ ...doc,
+ id: this.id,
+ },
+ }));
+ }
+ } else if (msg.type === "document") {
+ this.dispatchTypedEvent("document", new CustomEvent("document", {
+ detail: {
+ ...msg.data,
+ id: this.id,
+ },
+ }));
+ }
+ }
+
+ this.ws.onmessage = ((ev: MessageEvent) => {
+ if (typeof ev.data !== "string") {
+ this.ws.close();
+ return;
+ }
+ try {
+ const msg = JSON.parse(ev.data) as Message;
+ messageHandler(msg);
+ } catch (error) {
+ console.error("Error on message", error);
+ }
+ }).bind(this);
+
+ this.ws.onclose = ((ev: CloseEvent) => {
+ try {
+ const msg = JSON.parse(ev.reason) as Message;
+ messageHandler(msg);
+ } catch (error) {
+ console.error("Error on close", error);
+ }
+ }).bind(this);
+
+ this.ws.onerror = ((_: Event) => {
+ this.status = "failed"
+ this.dispatchTypedEvent("error", new CustomEvent("error", {
+ detail: {
+ status: this.status,
+ data: this.data,
+ error: "WebSocket error",
+ id: this.id,
+ },
+ }));
+ }).bind(this);
+ }
+
+ close() {
+ this.ws.close();
+ }
+}
diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts
index f93bb48e3..e068a269c 100644
--- a/apps/js-sdk/firecrawl/src/index.ts
+++ b/apps/js-sdk/firecrawl/src/index.ts
@@ -1,2146 +1,25 @@
-import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios";
-import * as zt from "zod";
-import { zodToJsonSchema } from "zod-to-json-schema";
-import { TypedEventTarget } from "typescript-event-target";
+// Unified entrypoint for Firecrawl JS SDK
+// - Default export: unified Firecrawl (v2 by default, v1 under .v1)
+// - Named export FirecrawlClient: direct v2 client
+// - Named export FirecrawlAppV1: legacy v1 class
+// - Re-export v2 types
-/**
- * Configuration interface for FirecrawlApp.
- * @param apiKey - Optional API key for authentication.
- * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
- */
-export interface FirecrawlAppConfig {
- apiKey?: string | null;
- apiUrl?: string | null;
-}
-
-/**
- * Metadata for a Firecrawl document.
- * Includes various optional properties for document metadata.
- */
-export interface FirecrawlDocumentMetadata {
- title?: string;
- description?: string;
- language?: string;
- keywords?: string;
- robots?: string;
- ogTitle?: string;
- ogDescription?: string;
- ogUrl?: string;
- ogImage?: string;
- ogAudio?: string;
- ogDeterminer?: string;
- ogLocale?: string;
- ogLocaleAlternate?: string[];
- ogSiteName?: string;
- ogVideo?: string;
- dctermsCreated?: string;
- dcDateCreated?: string;
- dcDate?: string;
- dctermsType?: string;
- dcType?: string;
- dctermsAudience?: string;
- dctermsSubject?: string;
- dcSubject?: string;
- dcDescription?: string;
- dctermsKeywords?: string;
- modifiedTime?: string;
- publishedTime?: string;
- articleTag?: string;
- articleSection?: string;
- sourceURL?: string;
- statusCode?: number;
- error?: string;
- proxyUsed?: "basic" | "stealth";
- cacheState?: "miss" | "hit";
- cachedAt?: string;
- [key: string]: any; // Allows for additional metadata properties not explicitly defined.
-}
-
-/**
- * Document interface for Firecrawl.
- * Represents a document retrieved or processed by Firecrawl.
- */
-export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | never) = never> {
- url?: string;
- markdown?: string;
- html?: string;
- rawHtml?: string;
- links?: string[];
- extract?: T;
- json?: T;
- screenshot?: string;
- metadata?: FirecrawlDocumentMetadata;
- actions: ActionsSchema;
- changeTracking?: {
- previousScrapeAt: string | null;
- changeStatus: "new" | "same" | "changed" | "removed";
- visibility: "visible" | "hidden";
- diff?: {
- text: string;
- json: {
- files: Array<{
- from: string | null;
- to: string | null;
- chunks: Array<{
- content: string;
- changes: Array<{
- type: string;
- normal?: boolean;
- ln?: number;
- ln1?: number;
- ln2?: number;
- content: string;
- }>;
- }>;
- }>;
- };
- };
- json?: any;
- };
- // v1 search only
- title?: string;
- description?: string;
-}
-
-/**
- * Parameters for scraping operations.
- * Defines the options and configurations available for scraping web content.
- */
-export interface CrawlScrapeOptions {
- formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "changeTracking")[];
- headers?: Record<string, string>;
- includeTags?: string[];
- excludeTags?: string[];
- onlyMainContent?: boolean;
- waitFor?: number;
- timeout?: number;
- location?: {
- country?: string;
- languages?: string[];
- };
- mobile?: boolean;
- skipTlsVerification?: boolean;
- removeBase64Images?: boolean;
- blockAds?: boolean;
- proxy?: "basic" | "stealth" | "auto";
- storeInCache?: boolean;
- maxAge?: number;
- parsePDF?: boolean;
-}
-
-export type Action = {
- type: "wait",
- milliseconds?: number,
- selector?: string,
-} | {
- type: "click",
- selector: string,
- all?: boolean,
-} | {
- type: "screenshot",
- fullPage?: boolean,
- quality?: number,
-} | {
- type: "write",
- text: string,
-} | {
- type: "press",
- key: string,
-} | {
- type: "scroll",
- direction?: "up" | "down",
- selector?: string,
-} | {
- type: "scrape",
-} | {
- type: "executeJavascript",
- script: string,
-};
-
-export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema extends (Action[] | undefined) = undefined> extends CrawlScrapeOptions {
- extract?: {
- prompt?: string;
- schema?: LLMSchema;
- systemPrompt?: string;
- };
- jsonOptions?:{
- prompt?: string;
- schema?: LLMSchema;
- systemPrompt?: string;
- }
- changeTrackingOptions?: {
- prompt?: string;
- schema?: any;
- modes?: ("json" | "git-diff")[];
- tag?: string | null;
- }
- actions?: ActionsSchema;
- agent?: AgentOptions;
- zeroDataRetention?: boolean;
-}
-
-export interface ActionsResult {
- screenshots: string[];
- scrapes: ({
- url: string;
- html: string;
- })[];
- javascriptReturns: {
- type: string;
- value: unknown
- }[];
-}
-
-/**
- * Response interface for scraping operations.
- * Defines the structure of the response received after a scraping operation.
- */
-export interface ScrapeResponse<LLMResult = any, ActionsSchema extends (ActionsResult | never) = never> extends FirecrawlDocument<LLMResult, ActionsSchema> {
- success: true;
- warning?: string;
- error?: string;
-}
-
-/**
- * Parameters for crawling operations.
- * Includes options for both scraping and mapping during a crawl.
- */
-export interface CrawlParams {
- includePaths?: string[];
- excludePaths?: string[];
- maxDepth?: number;
- maxDiscoveryDepth?: number;
- limit?: number;
- allowBackwardLinks?: boolean;
- crawlEntireDomain?: boolean;
- allowExternalLinks?: boolean;
- ignoreSitemap?: boolean;
- scrapeOptions?: CrawlScrapeOptions;
- webhook?: string | {
- url: string;
- headers?: Record<string, string>;
- metadata?: Record<string, string>;
- events?: ["completed", "failed", "page", "started"][number][];
- };
- deduplicateSimilarURLs?: boolean;
- ignoreQueryParameters?: boolean;
- regexOnFullURL?: boolean;
- /**
- * Delay in seconds between scrapes. This helps respect website rate limits.
- * If not provided, the crawler may use the robots.txt crawl delay if available.
- */
- delay?: number;
- allowSubdomains?: boolean;
- maxConcurrency?: number;
- zeroDataRetention?: boolean;
-}
-
-/**
- * Response interface for crawling operations.
- * Defines the structure of the response received after initiating a crawl.
- */
-export interface CrawlResponse {
- id?: string;
- url?: string;
- success: true;
- error?: string;
-}
-
-/**
- * Response interface for batch scrape operations.
- * Defines the structure of the response received after initiating a crawl.
- */
-export interface BatchScrapeResponse {
- id?: string;
- url?: string;
- success: true;
- error?: string;
- invalidURLs?: string[];
-}
-
-/**
- * Response interface for job status checks.
- * Provides detailed status of a crawl job including progress and results.
- */
-export interface CrawlStatusResponse {
- success: true;
- status: "scraping" | "completed" | "failed" | "cancelled";
- completed: number;
- total: number;
- creditsUsed: number;
- expiresAt: Date;
- next?: string;
- data: FirecrawlDocument<undefined>[];
-};
-
-/**
- * Response interface for batch scrape job status checks.
- * Provides detailed status of a batch scrape job including progress and results.
- */
-export interface BatchScrapeStatusResponse {
- success: true;
- status: "scraping" | "completed" | "failed" | "cancelled";
- completed: number;
- total: number;
- creditsUsed: number;
- expiresAt: Date;
- next?: string;
- data: FirecrawlDocument<undefined>[];
-};
-
-/**
- * Parameters for mapping operations.
- * Defines options for mapping URLs during a crawl.
- */
-export interface MapParams {
- search?: string;
- ignoreSitemap?: boolean;
- includeSubdomains?: boolean;
- sitemapOnly?: boolean;
- limit?: number;
- timeout?: number;
- useIndex?: boolean;
-}
+export { FirecrawlClient } from "./v2/client";
+export * from "./v2/types";
+export { default as FirecrawlAppV1 } from "./v1";
-/**
- * Response interface for mapping operations.
- * Defines the structure of the response received after a mapping operation.
- */
-export interface MapResponse {
- success: true;
- links?: string[];
- error?: string;
-}
-
-/**
- * Parameters for extracting information from URLs.
- * Defines options for extracting information from URLs.
- */
-export interface AgentOptions {
- model?: string;
- prompt?: string;
- sessionId?: string;
-}
+import V1 from "./v1";
+import { FirecrawlClient as V2 } from "./v2/client";
+import type { FirecrawlAppConfig } from "./v1";
-/**
- * Parameters for extracting information from URLs.
- * Defines options for extracting information from URLs.
- */
-export interface AgentOptionsExtract {
- model?: string;
- sessionId?: string;
-}
+export class Firecrawl extends V2 {
+ public v1: V1;
-export interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
- prompt?: string;
- schema?: LLMSchema | object;
- systemPrompt?: string;
- allowExternalLinks?: boolean;
- enableWebSearch?: boolean;
- includeSubdomains?: boolean;
- origin?: string;
- showSources?: boolean;
- scrapeOptions?: CrawlScrapeOptions;
- agent?: AgentOptionsExtract;
-}
-
-/**
- * Response interface for extracting information from URLs.
- * Defines the structure of the response received after extracting information from URLs.
- */
-export interface ExtractResponse<LLMSchema extends zt.ZodSchema = any> {
- success: boolean;
- data: LLMSchema;
- error?: string;
- warning?: string;
- sources?: string[];
-}
-
-/**
- * Error response interface.
- * Defines the structure of the response received when an error occurs.
- */
-export interface ErrorResponse {
- success: false;
- error: string;
-}
-
-/**
- * Custom error class for Firecrawl.
- * Extends the built-in Error class to include a status code.
- */
-export class FirecrawlError extends Error {
- statusCode: number;
- details?: any;
- constructor(message: string, statusCode: number, details?: any) {
- super(message);
- this.statusCode = statusCode;
- this.details = details;
+ constructor(opts: FirecrawlAppConfig = {}) {
+ super(opts as any);
+ this.v1 = new V1(opts);
}
}
-/**
- * Parameters for search operations.
- * Defines options for searching and scraping search results.
- */
-export interface SearchParams {
- limit?: number;
- tbs?: string;
- filter?: string;
- lang?: string;
- country?: string;
- location?: string;
- origin?: string;
- timeout?: number;
- scrapeOptions?: ScrapeParams;
-}
-
-/**
- * Response interface for search operations.
- * Defines the structure of the response received after a search operation.
- */
-export interface SearchResponse {
- success: boolean;
- data: FirecrawlDocument<undefined>[];
- warning?: string;
- error?: string;
-}
-
-/**
- * Response interface for crawl/batch scrape error monitoring.
- */
-export interface CrawlErrorsResponse {
- /**
- * Scrapes that errored out + error details
- */
- errors: {
- id: string,
- timestamp?: string,
- url: string,
- error: string,
- }[];
-
- /**
- * URLs blocked by robots.txt
- */
- robotsBlocked: string[];
-};
-
-/**
- * Parameters for deep research operations.
- * Defines options for conducting deep research on a query.
- */
-export interface DeepResearchParams<LLMSchema extends zt.ZodSchema = any> {
- /**
- * Maximum depth of research iterations (1-10)
- * @default 7
- */
- maxDepth?: number;
- /**
- * Time limit in seconds (30-300)
- * @default 270
- */
- timeLimit?: number;
- /**
- * Maximum number of URLs to analyze (1-1000)
- * @default 20
- */
- maxUrls?: number;
- /**
- * The prompt to use for the final analysis
- */
- analysisPrompt?: string;
- /**
- * The system prompt to use for the research agent
- */
- systemPrompt?: string;
- /**
- * The formats to use for the final analysis
- */
- formats?: ("markdown" | "json")[];
- /**
- * The JSON options to use for the final analysis
- */
- jsonOptions?:{
- prompt?: string;
- schema?: LLMSchema;
- systemPrompt?: string;
- };
- /**
- * Experimental flag for streaming steps
- */
- // __experimental_streamSteps?: boolean;
-}
-
-/**
- * Response interface for deep research operations.
- */
-export interface DeepResearchResponse {
- success: boolean;
- id: string;
-}
-
-/**
- * Status response interface for deep research operations.
- */
-export interface DeepResearchStatusResponse {
- success: boolean;
- data: {
- finalAnalysis: string;
- activities: Array<{
- type: string;
- status: string;
- message: string;
- timestamp: string;
- depth: number;
- }>;
- sources: Array<{
- url: string;
- title: string;
- description: string;
- }>;
- };
- status: "processing" | "completed" | "failed";
- error?: string;
- expiresAt: string;
- currentDepth: number;
- maxDepth: number;
- activities: Array<{
- type: string;
- status: string;
- message: string;
- timestamp: string;
- depth: number;
- }>;
- sources: Array<{
- url: string;
- title: string;
- description: string;
- }>;
- summaries: string[];
-}
-
-/**
- * Parameters for LLMs.txt generation operations.
- */
-export interface GenerateLLMsTextParams {
- /**
- * Maximum number of URLs to process (1-100)
- * @default 10
- */
- maxUrls?: number;
- /**
- * Whether to show the full LLMs-full.txt in the response
- * @default false
- */
- showFullText?: boolean;
- /**
- * Whether to use cached content if available
- * @default true
- */
- cache?: boolean;
- /**
- * Experimental flag for streaming
- */
- __experimental_stream?: boolean;
-}
-
-/**
- * Response interface for LLMs.txt generation operations.
- */
-export interface GenerateLLMsTextResponse {
- success: boolean;
- id: string;
-}
-
-/**
- * Status response interface for LLMs.txt generation operations.
- */
-export interface GenerateLLMsTextStatusResponse {
- success: boolean;
- data: {
- llmstxt: string;
- llmsfulltxt?: string;
- };
- status: "processing" | "completed" | "failed";
- error?: string;
- expiresAt: string;
-}
-
-/**
- * Main class for interacting with the Firecrawl API.
- * Provides methods for scraping, searching, crawling, and mapping web content.
- */
-export default class FirecrawlApp {
- public apiKey: string;
- public apiUrl: string;
- public version: string = "1.25.1";
-
- private isCloudService(url: string): boolean {
- return url.includes('api.firecrawl.dev');
- }
-
- private async getVersion(): Promise<string> {
- try {
- const packageJson = await import('../package.json', { assert: { type: 'json' } });
- return packageJson.default.version;
- } catch (error) {
- console.error("Error getting version:", error);
- return "1.25.1";
- }
- }
-
- private async init() {
- this.version = await this.getVersion();
- }
-
- /**
- * Initializes a new instance of the FirecrawlApp class.
- * @param config - Configuration options for the FirecrawlApp instance.
- */
- constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
- const baseUrl = apiUrl || "https://api.firecrawl.dev";
-
- if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
- throw new FirecrawlError("No API key provided", 401);
- }
-
- this.apiKey = apiKey || '';
- this.apiUrl = baseUrl;
- this.init();
- }
-
- /**
- * Scrapes a URL using the Firecrawl API.
- * @param url - The URL to scrape.
- * @param params - Additional parameters for the scrape request.
- * @returns The response from the scrape operation.
- */
- async scrapeUrl<T extends zt.ZodSchema, ActionsSchema extends (Action[] | undefined) = undefined>(
- url: string,
- params?: ScrapeParams<T, ActionsSchema>
- ): Promise<ScrapeResponse<zt.infer<T>, ActionsSchema extends Action[] ? ActionsResult : never> | ErrorResponse> {
- const headers: AxiosRequestHeaders = {
- "Content-Type": "application/json",
- Authorization: `Bearer ${this.apiKey}`,
- } as AxiosRequestHeaders;
- let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
- if (jsonData?.extract?.schema) {
- let schema = jsonData.extract.schema;
-
- // Try parsing the schema as a Zod schema
- try {
- schema = zodToJsonSchema(schema);
- } catch (error) {
-
- }
- jsonData = {
- ...jsonData,
- extract: {
- ...jsonData.extract,
- schema: schema,
- },
- };
- }
-
- if (jsonData?.jsonOptions?.schema) {
- let schema = jsonData.jsonOptions.schema;
- // Try parsing the schema as a Zod schema
- try {
- schema = zodToJsonSchema(schema);
- } catch (error) {
-
- }
- jsonData = {
- ...jsonData,
- jsonOptions: {
- ...jsonData.jsonOptions,
- schema: schema,
- },
- };
- }
- try {
- const response: AxiosResponse = await axios.post(
- this.apiUrl + `/v1/scrape`,
- jsonData,
- { headers, timeout: params?.timeout !== undefined ? (params.timeout + 5000) : undefined },
- );
- if (response.status === 200) {
- const responseData = response.data;
- if (responseData.success) {
- return {
- success: true,
- warning: responseData.warning,
- error: responseData.error,
- ...responseData.data
- };
- } else {
- throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
- }
- } else {
- this.handleError(response, "scrape URL");
- }
- } catch (error: any) {
- this.handleError(error.response, "scrape URL");
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Searches using the Firecrawl API and optionally scrapes the results.
- * @param query - The search query string.
- * @param params - Optional parameters for the search request.
- * @returns The response from the search operation.
- */
- async search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse> {
- const headers: AxiosRequestHeaders = {
- "Content-Type": "application/json",
- Authorization: `Bearer ${this.apiKey}`,
- } as AxiosRequestHeaders;
-
- let jsonData: any = {
- query,
- limit: params?.limit ?? 5,
- tbs: params?.tbs,
- filter: params?.filter,
- lang: params?.lang ?? "en",
- country: params?.country ?? "us",
- location: params?.location,
- origin: `js-sdk@${this.version}`,
- timeout: params?.timeout ?? 60000,
- scrapeOptions: params?.scrapeOptions ?? { formats: [] },
- };
-
- if (jsonData?.scrapeOptions?.extract?.schema) {
- let schema = jsonData.scrapeOptions.extract.schema;
-
- // Try parsing the schema as a Zod schema
- try {
- schema = zodToJsonSchema(schema);
- } catch (error) {
-
- }
- jsonData = {
- ...jsonData,
- scrapeOptions: {
- ...jsonData.scrapeOptions,
- extract: {
- ...jsonData.scrapeOptions.extract,
- schema: schema,
- },
- },
- };
- }
-
- try {
- const response: AxiosResponse = await this.postRequest(
- this.apiUrl + `/v1/search`,
- jsonData,
- headers
- );
-
- if (response.status === 200) {
- const responseData = response.data;
- if (responseData.success) {
- return {
- success: true,
- data: responseData.data as FirecrawlDocument<any>[],
- warning: responseData.warning,
- };
- } else {
- throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);
- }
- } else {
- this.handleError(response, "search");
- }
- } catch (error: any) {
- if (error.response?.data?.error) {
- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
- } else {
- throw new FirecrawlError(error.message, 500);
- }
- }
- return { success: false, error: "Internal server error.", data: [] };
- }
-
- /**
- * Initiates a crawl job for a URL using the Firecrawl API.
- * @param url - The URL to crawl.
- * @param params - Additional parameters for the crawl request.
- * @param pollInterval - Time in seconds for job status checks.
- * @param idempotencyKey - Optional idempotency key for the request.
- * @returns The response from the crawl operation.
- */
- async crawlUrl(
- url: string,
- params?: CrawlParams,
- pollInterval: number = 2,
- idempotencyKey?: string
- ): Promise<CrawlStatusResponse | ErrorResponse> {
- const headers = this.prepareHeaders(idempotencyKey);
- let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
- try {
- const response: AxiosResponse = await this.postRequest(
- this.apiUrl + `/v1/crawl`,
- jsonData,
- headers
- );
- if (response.status === 200) {
- const id: string = response.data.id;
- return this.monitorJobStatus(id, headers, pollInterval);
- } else {
- this.handleError(response, "start crawl job");
- }
- } catch (error: any) {
- if (error.response?.data?.error) {
- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
- } else {
- throw new FirecrawlError(error.message, 500);
- }
- }
- return { success: false, error: "Internal server error." };
- }
-
- async asyncCrawlUrl(
- url: string,
- params?: CrawlParams,
- idempotencyKey?: string
- ): Promise<CrawlResponse | ErrorResponse> {
- const headers = this.prepareHeaders(idempotencyKey);
- let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
- try {
- const response: AxiosResponse = await this.postRequest(
- this.apiUrl + `/v1/crawl`,
- jsonData,
- headers
- );
- if (response.status === 200) {
- return response.data;
- } else {
- this.handleError(response, "start crawl job");
- }
- } catch (error: any) {
- if (error.response?.data?.error) {
- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
- } else {
- throw new FirecrawlError(error.message, 500);
- }
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Checks the status of a crawl job using the Firecrawl API.
- * @param id - The ID of the crawl operation.
- * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
- * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
- * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
- * @param limit - How many entries to return. Only used when `getAllData = false`.
- * @returns The response containing the job status.
- */
- async checkCrawlStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse> {
- if (!id) {
- throw new FirecrawlError("No crawl ID provided", 400);
- }
+export default Firecrawl;
- const headers: AxiosRequestHeaders = this.prepareHeaders();
- const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
- if (skip !== undefined) {
- targetURL.searchParams.set("skip", skip.toString());
- }
- if (limit !== undefined) {
- targetURL.searchParams.set("limit", limit.toString());
- }
-
- try {
- const response: AxiosResponse = await this.getRequest(
- targetURL.href,
- headers
- );
- if (response.status === 200) {
- let allData = response.data.data;
- if (getAllData && response.data.status === "completed") {
- let statusData = response.data
- if ("data" in statusData) {
- let data = statusData.data;
- while (typeof statusData === 'object' && 'next' in statusData) {
- if (data.length === 0) {
- break
- }
- statusData = (await this.getRequest(statusData.next, headers)).data;
- data = data.concat(statusData.data);
- }
- allData = data;
- }
- }
-
- let resp: CrawlStatusResponse | ErrorResponse = {
- success: response.data.success,
- status: response.data.status,
- total: response.data.total,
- completed: response.data.completed,
- creditsUsed: response.data.creditsUsed,
- next: getAllData ? undefined : response.data.next,
- expiresAt: new Date(response.data.expiresAt),
- data: allData
- }
-
- if (!response.data.success && response.data.error) {
- resp = {
- ...resp,
- success: false,
- error: response.data.error
- } as ErrorResponse;
- }
-
- if (response.data.next) {
- (resp as CrawlStatusResponse).next = response.data.next;
- }
-
- return resp;
- } else {
- this.handleError(response, "check crawl status");
- }
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500);
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Returns information about crawl errors.
- * @param id - The ID of the crawl operation.
- * @returns Information about crawl errors.
- */
- async checkCrawlErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {
- const headers = this.prepareHeaders();
- try {
- const response: AxiosResponse = await this.deleteRequest(
- `${this.apiUrl}/v1/crawl/${id}/errors`,
- headers
- );
- if (response.status === 200) {
- return response.data;
- } else {
- this.handleError(response, "check crawl errors");
- }
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500);
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Cancels a crawl job using the Firecrawl API.
- * @param id - The ID of the crawl operation.
- * @returns The response from the cancel crawl operation.
- */
- async cancelCrawl(id: string): Promise<ErrorResponse> {
- const headers = this.prepareHeaders();
- try {
- const response: AxiosResponse = await this.deleteRequest(
- `${this.apiUrl}/v1/crawl/${id}`,
- headers
- );
- if (response.status === 200) {
- return response.data;
- } else {
- this.handleError(response, "cancel crawl job");
- }
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500);
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Initiates a crawl job and returns a CrawlWatcher to monitor the job via WebSocket.
- * @param url - The URL to crawl.
- * @param params - Additional parameters for the crawl request.
- * @param idempotencyKey - Optional idempotency key for the request.
- * @returns A CrawlWatcher instance to monitor the crawl job.
- */
- async crawlUrlAndWatch(
- url: string,
- params?: CrawlParams,
- idempotencyKey?: string,
- ) {
- const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
-
- if (crawl.success && crawl.id) {
- const id = crawl.id;
- return new CrawlWatcher(id, this);
- }
-
- throw new FirecrawlError("Crawl job failed to start", 400);
- }
-
- /**
- * Maps a URL using the Firecrawl API.
- * @param url - The URL to map.
- * @param params - Additional parameters for the map request.
- * @returns The response from the map operation.
- */
- async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
- const headers = this.prepareHeaders();
- let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
-
- try {
- const response: AxiosResponse = await this.postRequest(
- this.apiUrl + `/v1/map`,
- jsonData,
- headers
- );
- if (response.status === 200) {
- return response.data as MapResponse;
- } else {
- this.handleError(response, "map");
- }
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500);
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Initiates a batch scrape job for multiple URLs using the Firecrawl API.
- * @param url - The URLs to scrape.
- * @param params - Additional parameters for the scrape request.
- * @param pollInterval - Time in seconds for job status checks.
- * @param idempotencyKey - Optional idempotency key for the request.
- * @param webhook - Optional webhook for the batch scrape.
- * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
- * @returns The response from the crawl operation.
- */
- async batchScrapeUrls(
- urls: string[],
- params?: ScrapeParams,
- pollInterval: number = 2,
- idempotencyKey?: string,
- webhook?: CrawlParams["webhook"],
- ignoreInvalidURLs?: boolean,
- maxConcurrency?: number,
- ): Promise<BatchScrapeStatusResponse | ErrorResponse> {
- const headers = this.prepareHeaders(idempotencyKey);
- let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: `js-sdk@${this.version}` };
- if (jsonData?.extract?.schema) {
- let schema = jsonData.extract.schema;
-
- // Try parsing the schema as a Zod schema
- try {
- schema = zodToJsonSchema(schema);
- } catch (error) {
-
- }
- jsonData = {
- ...jsonData,
- extract: {
- ...jsonData.extract,
- schema: schema,
- },
- };
- }
- if (jsonData?.jsonOptions?.schema) {
- let schema = jsonData.jsonOptions.schema;
-
- // Try parsing the schema as a Zod schema
- try {
- schema = zodToJsonSchema(schema);
- } catch (error) {
-
- }
- jsonData = {
- ...jsonData,
- jsonOptions: {
- ...jsonData.jsonOptions,
- schema: schema,
- },
- };
- }
- try {
- const response: AxiosResponse = await this.postRequest(
- this.apiUrl + `/v1/batch/scrape`,
- jsonData,
- headers
- );
- if (response.status === 200) {
- const id: string = response.data.id;
- return this.monitorJobStatus(id, headers, pollInterval);
- } else {
- this.handleError(response, "start batch scrape job");
- }
- } catch (error: any) {
- if (error.response?.data?.error) {
- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
- } else {
- throw new FirecrawlError(error.message, 500);
- }
- }
- return { success: false, error: "Internal server error." };
- }
-
- async asyncBatchScrapeUrls(
- urls: string[],
- params?: ScrapeParams,
- idempotencyKey?: string,
- webhook?: CrawlParams["webhook"],
- ignoreInvalidURLs?: boolean,
- ): Promise<BatchScrapeResponse | ErrorResponse> {
- const headers = this.prepareHeaders(idempotencyKey);
- let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
- try {
- const response: AxiosResponse = await this.postRequest(
- this.apiUrl + `/v1/batch/scrape`,
- jsonData,
- headers
- );
- if (response.status === 200) {
- return response.data;
- } else {
- this.handleError(response, "start batch scrape job");
- }
- } catch (error: any) {
- if (error.response?.data?.error) {
- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
- } else {
- throw new FirecrawlError(error.message, 500);
- }
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.
- * @param urls - The URL to scrape.
- * @param params - Additional parameters for the scrape request.
- * @param idempotencyKey - Optional idempotency key for the request.
- * @returns A CrawlWatcher instance to monitor the crawl job.
- */
- async batchScrapeUrlsAndWatch(
- urls: string[],
- params?: ScrapeParams,
- idempotencyKey?: string,
- webhook?: CrawlParams["webhook"],
- ignoreInvalidURLs?: boolean,
- ) {
- const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
-
- if (crawl.success && crawl.id) {
- const id = crawl.id;
- return new CrawlWatcher(id, this);
- }
-
- throw new FirecrawlError("Batch scrape job failed to start", 400);
- }
-
- /**
- * Checks the status of a batch scrape job using the Firecrawl API.
- * @param id - The ID of the batch scrape operation.
- * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
- * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
- * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
- * @param limit - How many entries to return. Only used when `getAllData = false`.
- * @returns The response containing the job status.
- */
- async checkBatchScrapeStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse> {
- if (!id) {
- throw new FirecrawlError("No batch scrape ID provided", 400);
- }
-
- const headers: AxiosRequestHeaders = this.prepareHeaders();
- const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
- if (skip !== undefined) {
- targetURL.searchParams.set("skip", skip.toString());
- }
- if (limit !== undefined) {
- targetURL.searchParams.set("limit", limit.toString());
- }
-
- try {
- const response: AxiosResponse = await this.getRequest(
- targetURL.href,
- headers
- );
- if (response.status === 200) {
- let allData = response.data.data;
- if (getAllData && response.data.status === "completed") {
- let statusData = response.data
- if ("data" in statusData) {
- let data = statusData.data;
- while (typeof statusData === 'object' && 'next' in statusData) {
- if (data.length === 0) {
- break
- }
- statusData = (await this.getRequest(statusData.next, headers)).data;
- data = data.concat(statusData.data);
- }
- allData = data;
- }
- }
-
- let resp: BatchScrapeStatusResponse | ErrorResponse = {
- success: response.data.success,
- status: response.data.status,
- total: response.data.total,
- completed: response.data.completed,
- creditsUsed: response.data.creditsUsed,
- next: getAllData ? undefined : response.data.next,
- expiresAt: new Date(response.data.expiresAt),
- data: allData
- }
-
- if (!response.data.success && response.data.error) {
- resp = {
- ...resp,
- success: false,
- error: response.data.error
- } as ErrorResponse;
- }
-
- if (response.data.next) {
- (resp as BatchScrapeStatusResponse).next = response.data.next;
- }
-
- return resp;
- } else {
- this.handleError(response, "check batch scrape status");
- }
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500);
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Returns information about batch scrape errors.
- * @param id - The ID of the batch scrape operation.
- * @returns Information about batch scrape errors.
- */
- async checkBatchScrapeErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {
- const headers = this.prepareHeaders();
- try {
- const response: AxiosResponse = await this.deleteRequest(
- `${this.apiUrl}/v1/batch/scrape/${id}/errors`,
- headers
- );
- if (response.status === 200) {
- return response.data;
- } else {
- this.handleError(response, "check batch scrape errors");
- }
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500);
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Extracts information from URLs using the Firecrawl API.
- * Currently in Beta. Expect breaking changes on future minor versions.
- * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
- * @param params - Additional parameters for the extract request.
- * @returns The response from the extract operation.
- */
- async extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {
- const headers = this.prepareHeaders();
-
- let jsonData: { urls?: string[] } & ExtractParams<T> = { urls: urls, ...params };
- let jsonSchema: any;
- try {
- if (!params?.schema) {
- jsonSchema = undefined;
- } else {
- try {
- jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);
- } catch (_) {
- jsonSchema = params.schema;
- }
- }
- } catch (error: any) {
- throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
- }
-
- try {
- const response: AxiosResponse = await this.postRequest(
- this.apiUrl + `/v1/extract`,
- { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
- headers
- );
-
- if (response.status === 200) {
- const jobId = response.data.id;
- let extractStatus;
- do {
- const statusResponse: AxiosResponse = await this.getRequest(
- `${this.apiUrl}/v1/extract/${jobId}`,
- headers
- );
- extractStatus = statusResponse.data;
- if (extractStatus.status === "completed") {
- if (extractStatus.success) {
- return {
- success: true,
- data: extractStatus.data,
- warning: extractStatus.warning,
- error: extractStatus.error,
- sources: extractStatus?.sources || undefined,
- };
- } else {
- throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);
- }
- } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {
- throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);
- }
- await new Promise(resolve => setTimeout(resolve, 1000)); // Polling interval
- } while (extractStatus.status !== "completed");
- } else {
- this.handleError(response, "extract");
- }
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500, error.response?.data?.details);
- }
- return { success: false, error: "Internal server error."};
- }
-
- /**
- * Initiates an asynchronous extract job for a URL using the Firecrawl API.
- * @param url - The URL to extract data from.
- * @param params - Additional parameters for the extract request.
- * @param idempotencyKey - Optional idempotency key for the request.
- * @returns The response from the extract operation.
- */
- async asyncExtract(
- urls: string[],
- params?: ExtractParams,
- idempotencyKey?: string
- ): Promise<ExtractResponse | ErrorResponse> {
- const headers = this.prepareHeaders(idempotencyKey);
- let jsonData: any = { urls, ...params };
- let jsonSchema: any;
-
- try {
- if (!params?.schema) {
- jsonSchema = undefined;
- } else {
- try {
- jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);
- } catch (_) {
- jsonSchema = params.schema;
- }
- }
- } catch (error: any) {
- throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
- }
-
- try {
- const response: AxiosResponse = await this.postRequest(
- this.apiUrl + `/v1/extract`,
- { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
- headers
- );
-
- if (response.status === 200) {
- return response.data;
- } else {
- this.handleError(response, "start extract job");
- }
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500, error.response?.data?.details);
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Retrieves the status of an extract job.
- * @param jobId - The ID of the extract job.
- * @returns The status of the extract job.
- */
- async getExtractStatus(jobId: string): Promise<any> {
- try {
- const response: AxiosResponse = await this.getRequest(
- `${this.apiUrl}/v1/extract/${jobId}`,
- this.prepareHeaders()
- );
-
- if (response.status === 200) {
- return response.data;
- } else {
- this.handleError(response, "get extract status");
- }
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500);
- }
- }
-
- /**
- * Prepares the headers for an API request.
- * @param idempotencyKey - Optional key to ensure idempotency.
- * @returns The prepared headers.
- */
- prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
- return {
- "Content-Type": "application/json",
- Authorization: `Bearer ${this.apiKey}`,
- ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),
- } as AxiosRequestHeaders & { "x-idempotency-key"?: string };
- }
-
- /**
- * Sends a POST request to the specified URL.
- * @param url - The URL to send the request to.
- * @param data - The data to send in the request.
- * @param headers - The headers for the request.
- * @returns The response from the POST request.
- */
- postRequest(
- url: string,
- data: any,
- headers: AxiosRequestHeaders
- ): Promise<AxiosResponse> {
- return axios.post(url, data, { headers, timeout: (data?.timeout ? (data.timeout + 5000) : undefined) });
- }
-
- /**
- * Sends a GET request to the specified URL.
- * @param url - The URL to send the request to.
- * @param headers - The headers for the request.
- * @returns The response from the GET request.
- */
- async getRequest(
- url: string,
- headers: AxiosRequestHeaders
- ): Promise<AxiosResponse> {
- try {
- return await axios.get(url, { headers });
- } catch (error) {
- if (error instanceof AxiosError && error.response) {
- return error.response as AxiosResponse;
- } else {
- throw error;
- }
- }
- }
-
- /**
- * Sends a DELETE request to the specified URL.
- * @param url - The URL to send the request to.
- * @param headers - The headers for the request.
- * @returns The response from the DELETE request.
- */
- async deleteRequest(
- url: string,
- headers: AxiosRequestHeaders
- ): Promise<AxiosResponse> {
- try {
- return await axios.delete(url, { headers });
- } catch (error) {
- if (error instanceof AxiosError && error.response) {
- return error.response as AxiosResponse;
- } else {
- throw error;
- }
- }
- }
-
- /**
- * Monitors the status of a crawl job until completion or failure.
- * @param id - The ID of the crawl operation.
- * @param headers - The headers for the request.
- * @param checkInterval - Interval in seconds for job status checks.
- * @param checkUrl - Optional URL to check the status (used for v1 API)
- * @returns The final job status or data.
- */
- async monitorJobStatus(
- id: string,
- headers: AxiosRequestHeaders,
- checkInterval: number
- ): Promise<CrawlStatusResponse | ErrorResponse> {
- let failedTries = 0;
- let networkRetries = 0;
- const maxNetworkRetries = 3;
-
- while (true) {
- try {
- let statusResponse: AxiosResponse = await this.getRequest(
- `${this.apiUrl}/v1/crawl/${id}`,
- headers
- );
-
- if (statusResponse.status === 200) {
- failedTries = 0;
- networkRetries = 0;
- let statusData = statusResponse.data;
-
- if (statusData.status === "completed") {
- if ("data" in statusData) {
- let data = statusData.data;
- while (typeof statusData === 'object' && 'next' in statusData) {
- if (data.length === 0) {
- break
- }
- statusResponse = await this.getRequest(statusData.next, headers);
- statusData = statusResponse.data;
- data = data.concat(statusData.data);
- }
- statusData.data = data;
- return statusData;
- } else {
- throw new FirecrawlError("Crawl job completed but no data was returned", 500);
- }
- } else if (
- ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
- ) {
- checkInterval = Math.max(checkInterval, 2);
- await new Promise((resolve) =>
- setTimeout(resolve, checkInterval * 1000)
- );
- } else {
- throw new FirecrawlError(
- `Crawl job failed or was stopped. Status: ${statusData.status}`,
- 500
- );
- }
- } else {
- failedTries++;
- if (failedTries >= 3) {
- this.handleError(statusResponse, "check crawl status");
- }
- }
- } catch (error: any) {
- if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {
- networkRetries++;
- const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);
-
- await new Promise((resolve) => setTimeout(resolve, backoffDelay));
- continue;
- }
-
- throw new FirecrawlError(error, 500);
- }
- }
- }
-
- /**
- * Determines if an error is retryable (transient network error)
- * @param error - The error to check
- * @returns True if the error should be retried
- */
- private isRetryableError(error: any): boolean {
- if (error instanceof AxiosError) {
- if (!error.response) {
- const code = error.code;
- const message = error.message?.toLowerCase() || '';
-
- return (
- code === 'ECONNRESET' ||
- code === 'ETIMEDOUT' ||
- code === 'ENOTFOUND' ||
- code === 'ECONNREFUSED' ||
- message.includes('socket hang up') ||
- message.includes('network error') ||
- message.includes('timeout')
- );
- }
-
- if (error.response?.status === 408 || error.response?.status === 504) {
- return true;
- }
- }
-
- if (error && typeof error === 'object') {
- const code = error.code;
- const message = error.message?.toLowerCase() || '';
-
- if (code === 'ECONNRESET' ||
- code === 'ETIMEDOUT' ||
- code === 'ENOTFOUND' ||
- code === 'ECONNREFUSED' ||
- message.includes('socket hang up') ||
- message.includes('network error') ||
- message.includes('timeout')) {
- return true;
- }
-
- if (error.response?.status === 408 || error.response?.status === 504) {
- return true;
- }
- }
-
- return false;
- }
-
- /**
- * Handles errors from API responses.
- * @param {AxiosResponse} response - The response from the API.
- * @param {string} action - The action being performed when the error occurred.
- */
- handleError(response: AxiosResponse, action: string): void {
- if (!response) {
- throw new FirecrawlError(
- `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,
- 0
- );
- }
-
- if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
- const errorMessage: string =
- response.data.error || "Unknown error occurred";
- const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : '';
- throw new FirecrawlError(
- `Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}${details}`,
- response.status,
- response?.data?.details
- );
- } else {
- throw new FirecrawlError(
- `Unexpected error occurred while trying to ${action}. Status code: ${response.status}`,
- response.status
- );
- }
- }
-
- /**
- * Initiates a deep research operation on a given query and polls until completion.
- * @param query - The query to research.
- * @param params - Parameters for the deep research operation.
- * @param onActivity - Optional callback to receive activity updates in real-time.
- * @param onSource - Optional callback to receive source updates in real-time.
- * @returns The final research results.
- */
- async deepResearch(
- query: string,
- params: DeepResearchParams<zt.ZodSchema>,
- onActivity?: (activity: {
- type: string;
- status: string;
- message: string;
- timestamp: string;
- depth: number;
- }) => void,
- onSource?: (source: {
- url: string;
- title?: string;
- description?: string;
- icon?: string;
- }) => void
- ): Promise<DeepResearchStatusResponse | ErrorResponse> {
- try {
- const response = await this.asyncDeepResearch(query, params);
-
- if (!response.success || 'error' in response) {
- return { success: false, error: 'error' in response ? response.error : 'Unknown error' };
- }
-
- if (!response.id) {
- throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);
- }
-
- const jobId = response.id;
- let researchStatus;
- let lastActivityCount = 0;
- let lastSourceCount = 0;
-
- while (true) {
- researchStatus = await this.checkDeepResearchStatus(jobId);
-
- if ('error' in researchStatus && !researchStatus.success) {
- return researchStatus;
- }
-
- // Stream new activities through the callback if provided
- if (onActivity && researchStatus.activities) {
- const newActivities = researchStatus.activities.slice(lastActivityCount);
- for (const activity of newActivities) {
- onActivity(activity);
- }
- lastActivityCount = researchStatus.activities.length;
- }
-
- // Stream new sources through the callback if provided
- if (onSource && researchStatus.sources) {
- const newSources = researchStatus.sources.slice(lastSourceCount);
- for (const source of newSources) {
- onSource(source);
- }
- lastSourceCount = researchStatus.sources.length;
- }
-
- if (researchStatus.status === "completed") {
- return researchStatus;
- }
-
- if (researchStatus.status === "failed") {
- throw new FirecrawlError(
- `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,
- 500
- );
- }
-
- if (researchStatus.status !== "processing") {
- break;
- }
-
- await new Promise(resolve => setTimeout(resolve, 2000));
- }
-
- return { success: false, error: "Research job terminated unexpectedly" };
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500, error.response?.data?.details);
- }
- }
-
- /**
- * Initiates a deep research operation on a given query without polling.
- * @param params - Parameters for the deep research operation.
- * @returns The response containing the research job ID.
- */
- async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
- const headers = this.prepareHeaders();
- let jsonData: any = { query, ...params, origin: `js-sdk@${this.version}` };
-
- if (jsonData?.jsonOptions?.schema) {
- let schema = jsonData.jsonOptions.schema;
- // Try parsing the schema as a Zod schema
- try {
- schema = zodToJsonSchema(schema);
- } catch (error) {
- // Ignore error if schema can't be parsed as Zod
- }
- jsonData = {
- ...jsonData,
- jsonOptions: {
- ...jsonData.jsonOptions,
- schema: schema,
- },
- };
- }
-
- try {
- const response: AxiosResponse = await this.postRequest(
- `${this.apiUrl}/v1/deep-research`,
- jsonData,
- headers
- );
-
- if (response.status === 200) {
- return response.data;
- } else {
- this.handleError(response, "start deep research");
- }
- } catch (error: any) {
- if (error.response?.data?.error) {
- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
- } else {
- throw new FirecrawlError(error.message, 500);
- }
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Checks the status of a deep research operation.
- * @param id - The ID of the deep research operation.
- * @returns The current status and results of the research operation.
- */
- async checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {
- const headers = this.prepareHeaders();
- try {
- const response: AxiosResponse = await this.getRequest(
- `${this.apiUrl}/v1/deep-research/${id}`,
- headers
- );
-
- if (response.status === 200) {
- return response.data;
- } else if (response.status === 404) {
- throw new FirecrawlError("Deep research job not found", 404);
- } else {
- this.handleError(response, "check deep research status");
- }
- } catch (error: any) {
- if (error.response?.data?.error) {
- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
- } else {
- throw new FirecrawlError(error.message, 500);
- }
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * @deprecated Use deepResearch() instead
- * Initiates a deep research operation on a given topic and polls until completion.
- * @param topic - The topic to research.
- * @param params - Parameters for the deep research operation.
- * @param onActivity - Optional callback to receive activity updates in real-time.
- * @returns The final research results.
- */
- async __deepResearch(
- topic: string,
- params: DeepResearchParams,
- onActivity?: (activity: {
- type: string;
- status: string;
- message: string;
- timestamp: string;
- depth: number;
- }) => void
- ): Promise<DeepResearchStatusResponse | ErrorResponse> {
- try {
- const response = await this.__asyncDeepResearch(topic, params);
-
- if (!response.success || 'error' in response) {
- return { success: false, error: 'error' in response ? response.error : 'Unknown error' };
- }
-
- if (!response.id) {
- throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);
- }
-
- const jobId = response.id;
- let researchStatus;
- let lastActivityCount = 0;
-
- while (true) {
- researchStatus = await this.__checkDeepResearchStatus(jobId);
-
- if ('error' in researchStatus && !researchStatus.success) {
- return researchStatus;
- }
-
- // Stream new activities through the callback if provided
- if (onActivity && researchStatus.activities) {
- const newActivities = researchStatus.activities.slice(lastActivityCount);
- for (const activity of newActivities) {
- onActivity(activity);
- }
- lastActivityCount = researchStatus.activities.length;
- }
-
- if (researchStatus.status === "completed") {
- return researchStatus;
- }
-
- if (researchStatus.status === "failed") {
- throw new FirecrawlError(
- `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,
- 500
- );
- }
-
- if (researchStatus.status !== "processing") {
- break;
- }
-
- await new Promise(resolve => setTimeout(resolve, 2000));
- }
-
- return { success: false, error: "Research job terminated unexpectedly" };
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500, error.response?.data?.details);
- }
- }
-
- /**
- * @deprecated Use asyncDeepResearch() instead
- * Initiates a deep research operation on a given topic without polling.
- * @param params - Parameters for the deep research operation.
- * @returns The response containing the research job ID.
- */
- async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
- const headers = this.prepareHeaders();
- try {
- let jsonData: any = { topic, ...params, origin: `js-sdk@${this.version}` };
- const response: AxiosResponse = await this.postRequest(
- `${this.apiUrl}/v1/deep-research`,
- jsonData,
- headers
- );
-
- if (response.status === 200) {
- return response.data;
- } else {
- this.handleError(response, "start deep research");
- }
- } catch (error: any) {
- if (error.response?.data?.error) {
- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
- } else {
- throw new FirecrawlError(error.message, 500);
- }
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * @deprecated Use checkDeepResearchStatus() instead
- * Checks the status of a deep research operation.
- * @param id - The ID of the deep research operation.
- * @returns The current status and results of the research operation.
- */
- async __checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {
- const headers = this.prepareHeaders();
- try {
- const response: AxiosResponse = await this.getRequest(
- `${this.apiUrl}/v1/deep-research/${id}`,
- headers
- );
-
- if (response.status === 200) {
- return response.data;
- } else if (response.status === 404) {
- throw new FirecrawlError("Deep research job not found", 404);
- } else {
- this.handleError(response, "check deep research status");
- }
- } catch (error: any) {
- if (error.response?.data?.error) {
- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
- } else {
- throw new FirecrawlError(error.message, 500);
- }
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Generates LLMs.txt for a given URL and polls until completion.
- * @param url - The URL to generate LLMs.txt from.
- * @param params - Parameters for the LLMs.txt generation operation.
- * @returns The final generation results.
- */
- async generateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {
- try {
- const response = await this.asyncGenerateLLMsText(url, params);
-
- if (!response.success || 'error' in response) {
- return { success: false, error: 'error' in response ? response.error : 'Unknown error' };
- }
-
- if (!response.id) {
- throw new FirecrawlError(`Failed to start LLMs.txt generation. No job ID returned.`, 500);
- }
-
- const jobId = response.id;
- let generationStatus;
-
- while (true) {
- generationStatus = await this.checkGenerateLLMsTextStatus(jobId);
-
- if ('error' in generationStatus && !generationStatus.success) {
- return generationStatus;
- }
-
- if (generationStatus.status === "completed") {
- return generationStatus;
- }
-
- if (generationStatus.status === "failed") {
- throw new FirecrawlError(
- `LLMs.txt generation job ${generationStatus.status}. Error: ${generationStatus.error}`,
- 500
- );
- }
-
- if (generationStatus.status !== "processing") {
- break;
- }
-
- await new Promise(resolve => setTimeout(resolve, 2000));
- }
-
- return { success: false, error: "LLMs.txt generation job terminated unexpectedly" };
- } catch (error: any) {
- throw new FirecrawlError(error.message, 500, error.response?.data?.details);
- }
- }
-
- /**
- * Initiates a LLMs.txt generation operation without polling.
- * @param url - The URL to generate LLMs.txt from.
- * @param params - Parameters for the LLMs.txt generation operation.
- * @returns The response containing the generation job ID.
- */
- async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {
- const headers = this.prepareHeaders();
- let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
- try {
- const response: AxiosResponse = await this.postRequest(
- `${this.apiUrl}/v1/llmstxt`,
- jsonData,
- headers
- );
-
- if (response.status === 200) {
- return response.data;
- } else {
- this.handleError(response, "start LLMs.txt generation");
- }
- } catch (error: any) {
- if (error.response?.data?.error) {
- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
- } else {
- throw new FirecrawlError(error.message, 500);
- }
- }
- return { success: false, error: "Internal server error." };
- }
-
- /**
- * Checks the status of a LLMs.txt generation operation.
- * @param id - The ID of the LLMs.txt generation operation.
- * @returns The current status and results of the generation operation.
- */
- async checkGenerateLLMsTextStatus(id: string): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {
- const headers = this.prepareHeaders();
- try {
- const response: AxiosResponse = await this.getRequest(
- `${this.apiUrl}/v1/llmstxt/${id}`,
- headers
- );
-
- if (response.status === 200) {
- return response.data;
- } else if (response.status === 404) {
- throw new FirecrawlError("LLMs.txt generation job not found", 404);
- } else {
- this.handleError(response, "check LLMs.txt generation status");
- }
- } catch (error: any) {
- if (error.response?.data?.error) {
- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
- } else {
- throw new FirecrawlError(error.message, 500);
- }
- }
- return { success: false, error: "Internal server error." };
- }
-}
-
-interface CrawlWatcherEvents {
- document: CustomEvent<FirecrawlDocument<undefined>>,
- done: CustomEvent<{
- status: CrawlStatusResponse["status"];
- data: FirecrawlDocument<undefined>[];
- }>,
- error: CustomEvent<{
- status: CrawlStatusResponse["status"],
- data: FirecrawlDocument<undefined>[],
- error: string,
- }>,
-}
-
-export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
- private ws: WebSocket;
- public data: FirecrawlDocument<undefined>[];
- public status: CrawlStatusResponse["status"];
- public id: string;
-
- constructor(id: string, app: FirecrawlApp) {
- super();
- this.id = id;
- // replace `http` with `ws` (`http://` -> `ws://` and `https://` -> `wss://`)
- const wsUrl = app.apiUrl.replace(/^http/, "ws");
- this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
- this.status = "scraping";
- this.data = [];
-
- type ErrorMessage = {
- type: "error",
- error: string,
- }
-
- type CatchupMessage = {
- type: "catchup",
- data: CrawlStatusResponse,
- }
-
- type DocumentMessage = {
- type: "document",
- data: FirecrawlDocument<undefined>,
- }
-
- type DoneMessage = { type: "done" }
-
- type Message = ErrorMessage | CatchupMessage | DoneMessage | DocumentMessage;
-
- const messageHandler = (msg: Message) => {
- if (msg.type === "done") {
- this.status = "completed";
- this.dispatchTypedEvent("done", new CustomEvent("done", {
- detail: {
- status: this.status,
- data: this.data,
- id: this.id,
- },
- }));
- } else if (msg.type === "error") {
- this.status = "failed";
- this.dispatchTypedEvent("error", new CustomEvent("error", {
- detail: {
- status: this.status,
- data: this.data,
- error: msg.error,
- id: this.id,
- },
- }));
- } else if (msg.type === "catchup") {
- this.status = msg.data.status;
- this.data.push(...(msg.data.data ?? []));
- for (const doc of this.data) {
- this.dispatchTypedEvent("document", new CustomEvent("document", {
- detail: {
- ...doc,
- id: this.id,
- },
- }));
- }
- } else if (msg.type === "document") {
- this.dispatchTypedEvent("document", new CustomEvent("document", {
- detail: {
- ...msg.data,
- id: this.id,
- },
- }));
- }
- }
-
- this.ws.onmessage = ((ev: MessageEvent) => {
- if (typeof ev.data !== "string") {
- this.ws.close();
- return;
- }
- try {
- const msg = JSON.parse(ev.data) as Message;
- messageHandler(msg);
- } catch (error) {
- console.error("Error on message", error);
- }
- }).bind(this);
-
- this.ws.onclose = ((ev: CloseEvent) => {
- try {
- const msg = JSON.parse(ev.reason) as Message;
- messageHandler(msg);
- } catch (error) {
- console.error("Error on close", error);
- }
- }).bind(this);
-
- this.ws.onerror = ((_: Event) => {
- this.status = "failed"
- this.dispatchTypedEvent("error", new CustomEvent("error", {
- detail: {
- status: this.status,
- data: this.data,
- error: "WebSocket error",
- id: this.id,
- },
- }));
- }).bind(this);
- }
-
- close() {
- this.ws.close();
- }
-}
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
new file mode 100644
index 000000000..a54ae3c32
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -0,0 +1,2146 @@
+import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios";
+import * as zt from "zod";
+import { zodToJsonSchema } from "zod-to-json-schema";
+import { TypedEventTarget } from "typescript-event-target";
+
+/**
+ * Configuration interface for FirecrawlApp.
+ * @param apiKey - Optional API key for authentication.
+ * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
+ */
+export interface FirecrawlAppConfig {
+ apiKey?: string | null;
+ apiUrl?: string | null;
+}
+
+/**
+ * Metadata for a Firecrawl document.
+ * Includes various optional properties for document metadata.
+ */
+export interface FirecrawlDocumentMetadata {
+ title?: string;
+ description?: string;
+ language?: string;
+ keywords?: string;
+ robots?: string;
+ ogTitle?: string;
+ ogDescription?: string;
+ ogUrl?: string;
+ ogImage?: string;
+ ogAudio?: string;
+ ogDeterminer?: string;
+ ogLocale?: string;
+ ogLocaleAlternate?: string[];
+ ogSiteName?: string;
+ ogVideo?: string;
+ dctermsCreated?: string;
+ dcDateCreated?: string;
+ dcDate?: string;
+ dctermsType?: string;
+ dcType?: string;
+ dctermsAudience?: string;
+ dctermsSubject?: string;
+ dcSubject?: string;
+ dcDescription?: string;
+ dctermsKeywords?: string;
+ modifiedTime?: string;
+ publishedTime?: string;
+ articleTag?: string;
+ articleSection?: string;
+ sourceURL?: string;
+ statusCode?: number;
+ error?: string;
+ proxyUsed?: "basic" | "stealth";
+ cacheState?: "miss" | "hit";
+ cachedAt?: string;
+ [key: string]: any; // Allows for additional metadata properties not explicitly defined.
+}
+
+/**
+ * Document interface for Firecrawl.
+ * Represents a document retrieved or processed by Firecrawl.
+ */
+export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | never) = never> {
+ url?: string;
+ markdown?: string;
+ html?: string;
+ rawHtml?: string;
+ links?: string[];
+ extract?: T;
+ json?: T;
+ screenshot?: string;
+ metadata?: FirecrawlDocumentMetadata;
+ actions: ActionsSchema;
+ changeTracking?: {
+ previousScrapeAt: string | null;
+ changeStatus: "new" | "same" | "changed" | "removed";
+ visibility: "visible" | "hidden";
+ diff?: {
+ text: string;
+ json: {
+ files: Array<{
+ from: string | null;
+ to: string | null;
+ chunks: Array<{
+ content: string;
+ changes: Array<{
+ type: string;
+ normal?: boolean;
+ ln?: number;
+ ln1?: number;
+ ln2?: number;
+ content: string;
+ }>;
+ }>;
+ }>;
+ };
+ };
+ json?: any;
+ };
+ // v1 search only
+ title?: string;
+ description?: string;
+}
+
+/**
+ * Parameters for scraping operations.
+ * Defines the options and configurations available for scraping web content.
+ */
+export interface CrawlScrapeOptions {
+ formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "changeTracking")[];
+ headers?: Record<string, string>;
+ includeTags?: string[];
+ excludeTags?: string[];
+ onlyMainContent?: boolean;
+ waitFor?: number;
+ timeout?: number;
+ location?: {
+ country?: string;
+ languages?: string[];
+ };
+ mobile?: boolean;
+ skipTlsVerification?: boolean;
+ removeBase64Images?: boolean;
+ blockAds?: boolean;
+ proxy?: "basic" | "stealth" | "auto";
+ storeInCache?: boolean;
+ maxAge?: number;
+ parsePDF?: boolean;
+}
+
+export type Action = {
+ type: "wait",
+ milliseconds?: number,
+ selector?: string,
+} | {
+ type: "click",
+ selector: string,
+ all?: boolean,
+} | {
+ type: "screenshot",
+ fullPage?: boolean,
+ quality?: number,
+} | {
+ type: "write",
+ text: string,
+} | {
+ type: "press",
+ key: string,
+} | {
+ type: "scroll",
+ direction?: "up" | "down",
+ selector?: string,
+} | {
+ type: "scrape",
+} | {
+ type: "executeJavascript",
+ script: string,
+};
+
+export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema extends (Action[] | undefined) = undefined> extends CrawlScrapeOptions {
+ extract?: {
+ prompt?: string;
+ schema?: LLMSchema;
+ systemPrompt?: string;
+ };
+ jsonOptions?:{
+ prompt?: string;
+ schema?: LLMSchema;
+ systemPrompt?: string;
+ }
+ changeTrackingOptions?: {
+ prompt?: string;
+ schema?: any;
+ modes?: ("json" | "git-diff")[];
+ tag?: string | null;
+ }
+ actions?: ActionsSchema;
+ agent?: AgentOptions;
+ zeroDataRetention?: boolean;
+}
+
+export interface ActionsResult {
+ screenshots: string[];
+ scrapes: ({
+ url: string;
+ html: string;
+ })[];
+ javascriptReturns: {
+ type: string;
+ value: unknown
+ }[];
+}
+
+/**
+ * Response interface for scraping operations.
+ * Defines the structure of the response received after a scraping operation.
+ */
+export interface ScrapeResponse<LLMResult = any, ActionsSchema extends (ActionsResult | never) = never> extends FirecrawlDocument<LLMResult, ActionsSchema> {
+ success: true;
+ warning?: string;
+ error?: string;
+}
+
+/**
+ * Parameters for crawling operations.
+ * Includes options for both scraping and mapping during a crawl.
+ */
+export interface CrawlParams {
+ includePaths?: string[];
+ excludePaths?: string[];
+ maxDepth?: number;
+ maxDiscoveryDepth?: number;
+ limit?: number;
+ allowBackwardLinks?: boolean;
+ crawlEntireDomain?: boolean;
+ allowExternalLinks?: boolean;
+ ignoreSitemap?: boolean;
+ scrapeOptions?: CrawlScrapeOptions;
+ webhook?: string | {
+ url: string;
+ headers?: Record<string, string>;
+ metadata?: Record<string, string>;
+ events?: ["completed", "failed", "page", "started"][number][];
+ };
+ deduplicateSimilarURLs?: boolean;
+ ignoreQueryParameters?: boolean;
+ regexOnFullURL?: boolean;
+ /**
+ * Delay in seconds between scrapes. This helps respect website rate limits.
+ * If not provided, the crawler may use the robots.txt crawl delay if available.
+ */
+ delay?: number;
+ allowSubdomains?: boolean;
+ maxConcurrency?: number;
+ zeroDataRetention?: boolean;
+}
+
+/**
+ * Response interface for crawling operations.
+ * Defines the structure of the response received after initiating a crawl.
+ */
+export interface CrawlResponse {
+ id?: string;
+ url?: string;
+ success: true;
+ error?: string;
+}
+
+/**
+ * Response interface for batch scrape operations.
+ * Defines the structure of the response received after initiating a crawl.
+ */
+export interface BatchScrapeResponse {
+ id?: string;
+ url?: string;
+ success: true;
+ error?: string;
+ invalidURLs?: string[];
+}
+
+/**
+ * Response interface for job status checks.
+ * Provides detailed status of a crawl job including progress and results.
+ */
+export interface CrawlStatusResponse {
+ success: true;
+ status: "scraping" | "completed" | "failed" | "cancelled";
+ completed: number;
+ total: number;
+ creditsUsed: number;
+ expiresAt: Date;
+ next?: string;
+ data: FirecrawlDocument<undefined>[];
+};
+
+/**
+ * Response interface for batch scrape job status checks.
+ * Provides detailed status of a batch scrape job including progress and results.
+ */
+export interface BatchScrapeStatusResponse {
+ success: true;
+ status: "scraping" | "completed" | "failed" | "cancelled";
+ completed: number;
+ total: number;
+ creditsUsed: number;
+ expiresAt: Date;
+ next?: string;
+ data: FirecrawlDocument<undefined>[];
+};
+
+/**
+ * Parameters for mapping operations.
+ * Defines options for mapping URLs during a crawl.
+ */
+export interface MapParams {
+ search?: string;
+ ignoreSitemap?: boolean;
+ includeSubdomains?: boolean;
+ sitemapOnly?: boolean;
+ limit?: number;
+ timeout?: number;
+ useIndex?: boolean;
+}
+
+/**
+ * Response interface for mapping operations.
+ * Defines the structure of the response received after a mapping operation.
+ */
+export interface MapResponse {
+ success: true;
+ links?: string[];
+ error?: string;
+}
+
+/**
+ * Parameters for extracting information from URLs.
+ * Defines options for extracting information from URLs.
+ */
+export interface AgentOptions {
+ model?: string;
+ prompt?: string;
+ sessionId?: string;
+}
+
+/**
+ * Parameters for extracting information from URLs.
+ * Defines options for extracting information from URLs.
+ */
+export interface AgentOptionsExtract {
+ model?: string;
+ sessionId?: string;
+}
+
+export interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
+ prompt?: string;
+ schema?: LLMSchema | object;
+ systemPrompt?: string;
+ allowExternalLinks?: boolean;
+ enableWebSearch?: boolean;
+ includeSubdomains?: boolean;
+ origin?: string;
+ showSources?: boolean;
+ scrapeOptions?: CrawlScrapeOptions;
+ agent?: AgentOptionsExtract;
+}
+
+/**
+ * Response interface for extracting information from URLs.
+ * Defines the structure of the response received after extracting information from URLs.
+ */
+export interface ExtractResponse<LLMSchema extends zt.ZodSchema = any> {
+ success: boolean;
+ data: LLMSchema;
+ error?: string;
+ warning?: string;
+ sources?: string[];
+}
+
+/**
+ * Error response interface.
+ * Defines the structure of the response received when an error occurs.
+ */
+export interface ErrorResponse {
+ success: false;
+ error: string;
+}
+
+/**
+ * Custom error class for Firecrawl.
+ * Extends the built-in Error class to include a status code.
+ */
+export class FirecrawlError extends Error {
+ statusCode: number;
+ details?: any;
+ constructor(message: string, statusCode: number, details?: any) {
+ super(message);
+ this.statusCode = statusCode;
+ this.details = details;
+ }
+}
+
+/**
+ * Parameters for search operations.
+ * Defines options for searching and scraping search results.
+ */
+export interface SearchParams {
+ limit?: number;
+ tbs?: string;
+ filter?: string;
+ lang?: string;
+ country?: string;
+ location?: string;
+ origin?: string;
+ timeout?: number;
+ scrapeOptions?: ScrapeParams;
+}
+
+/**
+ * Response interface for search operations.
+ * Defines the structure of the response received after a search operation.
+ */
+export interface SearchResponse {
+ success: boolean;
+ data: FirecrawlDocument<undefined>[];
+ warning?: string;
+ error?: string;
+}
+
+/**
+ * Response interface for crawl/batch scrape error monitoring.
+ */
+export interface CrawlErrorsResponse {
+ /**
+ * Scrapes that errored out + error details
+ */
+ errors: {
+ id: string,
+ timestamp?: string,
+ url: string,
+ error: string,
+ }[];
+
+ /**
+ * URLs blocked by robots.txt
+ */
+ robotsBlocked: string[];
+};
+
+/**
+ * Parameters for deep research operations.
+ * Defines options for conducting deep research on a query.
+ */
+export interface DeepResearchParams<LLMSchema extends zt.ZodSchema = any> {
+ /**
+ * Maximum depth of research iterations (1-10)
+ * @default 7
+ */
+ maxDepth?: number;
+ /**
+ * Time limit in seconds (30-300)
+ * @default 270
+ */
+ timeLimit?: number;
+ /**
+ * Maximum number of URLs to analyze (1-1000)
+ * @default 20
+ */
+ maxUrls?: number;
+ /**
+ * The prompt to use for the final analysis
+ */
+ analysisPrompt?: string;
+ /**
+ * The system prompt to use for the research agent
+ */
+ systemPrompt?: string;
+ /**
+ * The formats to use for the final analysis
+ */
+ formats?: ("markdown" | "json")[];
+ /**
+ * The JSON options to use for the final analysis
+ */
+ jsonOptions?:{
+ prompt?: string;
+ schema?: LLMSchema;
+ systemPrompt?: string;
+ };
+ /**
+ * Experimental flag for streaming steps
+ */
+ // __experimental_streamSteps?: boolean;
+}
+
+/**
+ * Response interface for deep research operations.
+ */
+export interface DeepResearchResponse {
+ success: boolean;
+ id: string;
+}
+
+/**
+ * Status response interface for deep research operations.
+ */
+export interface DeepResearchStatusResponse {
+ success: boolean;
+ data: {
+ finalAnalysis: string;
+ activities: Array<{
+ type: string;
+ status: string;
+ message: string;
+ timestamp: string;
+ depth: number;
+ }>;
+ sources: Array<{
+ url: string;
+ title: string;
+ description: string;
+ }>;
+ };
+ status: "processing" | "completed" | "failed";
+ error?: string;
+ expiresAt: string;
+ currentDepth: number;
+ maxDepth: number;
+ activities: Array<{
+ type: string;
+ status: string;
+ message: string;
+ timestamp: string;
+ depth: number;
+ }>;
+ sources: Array<{
+ url: string;
+ title: string;
+ description: string;
+ }>;
+ summaries: string[];
+}
+
+/**
+ * Parameters for LLMs.txt generation operations.
+ */
+export interface GenerateLLMsTextParams {
+ /**
+ * Maximum number of URLs to process (1-100)
+ * @default 10
+ */
+ maxUrls?: number;
+ /**
+ * Whether to show the full LLMs-full.txt in the response
+ * @default false
+ */
+ showFullText?: boolean;
+ /**
+ * Whether to use cached content if available
+ * @default true
+ */
+ cache?: boolean;
+ /**
+ * Experimental flag for streaming
+ */
+ __experimental_stream?: boolean;
+}
+
+/**
+ * Response interface for LLMs.txt generation operations.
+ */
+export interface GenerateLLMsTextResponse {
+ success: boolean;
+ id: string;
+}
+
+/**
+ * Status response interface for LLMs.txt generation operations.
+ */
+export interface GenerateLLMsTextStatusResponse {
+ success: boolean;
+ data: {
+ llmstxt: string;
+ llmsfulltxt?: string;
+ };
+ status: "processing" | "completed" | "failed";
+ error?: string;
+ expiresAt: string;
+}
+
+/**
+ * Main class for interacting with the Firecrawl API.
+ * Provides methods for scraping, searching, crawling, and mapping web content.
+ */
+export default class FirecrawlApp {
+ public apiKey: string;
+ public apiUrl: string;
+ public version: string = "1.25.1";
+
+ private isCloudService(url: string): boolean {
+ return url.includes('api.firecrawl.dev');
+ }
+
+ private async getVersion(): Promise<string> {
+ try {
+ const packageJson = await import('../../package.json', { assert: { type: 'json' } });
+ return packageJson.default.version;
+ } catch (error) {
+ console.error("Error getting version:", error);
+ return "1.25.1";
+ }
+ }
+
+ private async init() {
+ this.version = await this.getVersion();
+ }
+
+ /**
+ * Initializes a new instance of the FirecrawlApp class.
+ * @param config - Configuration options for the FirecrawlApp instance.
+ */
+ constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
+ const baseUrl = apiUrl || "https://api.firecrawl.dev";
+
+ if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
+ throw new FirecrawlError("No API key provided", 401);
+ }
+
+ this.apiKey = apiKey || '';
+ this.apiUrl = baseUrl;
+ this.init();
+ }
+
+ /**
+ * Scrapes a URL using the Firecrawl API.
+ * @param url - The URL to scrape.
+ * @param params - Additional parameters for the scrape request.
+ * @returns The response from the scrape operation.
+ */
+ async scrapeUrl<T extends zt.ZodSchema, ActionsSchema extends (Action[] | undefined) = undefined>(
+ url: string,
+ params?: ScrapeParams<T, ActionsSchema>
+ ): Promise<ScrapeResponse<zt.infer<T>, ActionsSchema extends Action[] ? ActionsResult : never> | ErrorResponse> {
+ const headers: AxiosRequestHeaders = {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${this.apiKey}`,
+ } as AxiosRequestHeaders;
+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
+ if (jsonData?.extract?.schema) {
+ let schema = jsonData.extract.schema;
+
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+
+ }
+ jsonData = {
+ ...jsonData,
+ extract: {
+ ...jsonData.extract,
+ schema: schema,
+ },
+ };
+ }
+
+ if (jsonData?.jsonOptions?.schema) {
+ let schema = jsonData.jsonOptions.schema;
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+
+ }
+ jsonData = {
+ ...jsonData,
+ jsonOptions: {
+ ...jsonData.jsonOptions,
+ schema: schema,
+ },
+ };
+ }
+ try {
+ const response: AxiosResponse = await axios.post(
+ this.apiUrl + `/v1/scrape`,
+ jsonData,
+ { headers, timeout: params?.timeout !== undefined ? (params.timeout + 5000) : undefined },
+ );
+ if (response.status === 200) {
+ const responseData = response.data;
+ if (responseData.success) {
+ return {
+ success: true,
+ warning: responseData.warning,
+ error: responseData.error,
+ ...responseData.data
+ };
+ } else {
+ throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
+ }
+ } else {
+ this.handleError(response, "scrape URL");
+ }
+ } catch (error: any) {
+ this.handleError(error.response, "scrape URL");
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Searches using the Firecrawl API and optionally scrapes the results.
+ * @param query - The search query string.
+ * @param params - Optional parameters for the search request.
+ * @returns The response from the search operation.
+ */
+ async search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse> {
+ const headers: AxiosRequestHeaders = {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${this.apiKey}`,
+ } as AxiosRequestHeaders;
+
+ let jsonData: any = {
+ query,
+ limit: params?.limit ?? 5,
+ tbs: params?.tbs,
+ filter: params?.filter,
+ lang: params?.lang ?? "en",
+ country: params?.country ?? "us",
+ location: params?.location,
+ origin: `js-sdk@${this.version}`,
+ timeout: params?.timeout ?? 60000,
+ scrapeOptions: params?.scrapeOptions ?? { formats: [] },
+ };
+
+ if (jsonData?.scrapeOptions?.extract?.schema) {
+ let schema = jsonData.scrapeOptions.extract.schema;
+
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+
+ }
+ jsonData = {
+ ...jsonData,
+ scrapeOptions: {
+ ...jsonData.scrapeOptions,
+ extract: {
+ ...jsonData.scrapeOptions.extract,
+ schema: schema,
+ },
+ },
+ };
+ }
+
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/search`,
+ jsonData,
+ headers
+ );
+
+ if (response.status === 200) {
+ const responseData = response.data;
+ if (responseData.success) {
+ return {
+ success: true,
+ data: responseData.data as FirecrawlDocument<any>[],
+ warning: responseData.warning,
+ };
+ } else {
+ throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);
+ }
+ } else {
+ this.handleError(response, "search");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error.", data: [] };
+ }
+
+ /**
+ * Initiates a crawl job for a URL using the Firecrawl API.
+ * @param url - The URL to crawl.
+ * @param params - Additional parameters for the crawl request.
+ * @param pollInterval - Time in seconds for job status checks.
+ * @param idempotencyKey - Optional idempotency key for the request.
+ * @returns The response from the crawl operation.
+ */
+ async crawlUrl(
+ url: string,
+ params?: CrawlParams,
+ pollInterval: number = 2,
+ idempotencyKey?: string
+ ): Promise<CrawlStatusResponse | ErrorResponse> {
+ const headers = this.prepareHeaders(idempotencyKey);
+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/crawl`,
+ jsonData,
+ headers
+ );
+ if (response.status === 200) {
+ const id: string = response.data.id;
+ return this.monitorJobStatus(id, headers, pollInterval);
+ } else {
+ this.handleError(response, "start crawl job");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ async asyncCrawlUrl(
+ url: string,
+ params?: CrawlParams,
+ idempotencyKey?: string
+ ): Promise<CrawlResponse | ErrorResponse> {
+ const headers = this.prepareHeaders(idempotencyKey);
+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/crawl`,
+ jsonData,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start crawl job");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Checks the status of a crawl job using the Firecrawl API.
+ * @param id - The ID of the crawl operation.
+ * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+ * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+ * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
+ * @param limit - How many entries to return. Only used when `getAllData = false`.
+ * @returns The response containing the job status.
+ */
+ async checkCrawlStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse> {
+ if (!id) {
+ throw new FirecrawlError("No crawl ID provided", 400);
+ }
+
+ const headers: AxiosRequestHeaders = this.prepareHeaders();
+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+ if (skip !== undefined) {
+ targetURL.searchParams.set("skip", skip.toString());
+ }
+ if (limit !== undefined) {
+ targetURL.searchParams.set("limit", limit.toString());
+ }
+
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ targetURL.href,
+ headers
+ );
+ if (response.status === 200) {
+ let allData = response.data.data;
+ if (getAllData && response.data.status === "completed") {
+ let statusData = response.data
+ if ("data" in statusData) {
+ let data = statusData.data;
+ while (typeof statusData === 'object' && 'next' in statusData) {
+ if (data.length === 0) {
+ break
+ }
+ statusData = (await this.getRequest(statusData.next, headers)).data;
+ data = data.concat(statusData.data);
+ }
+ allData = data;
+ }
+ }
+
+ let resp: CrawlStatusResponse | ErrorResponse = {
+ success: response.data.success,
+ status: response.data.status,
+ total: response.data.total,
+ completed: response.data.completed,
+ creditsUsed: response.data.creditsUsed,
+ next: getAllData ? undefined : response.data.next,
+ expiresAt: new Date(response.data.expiresAt),
+ data: allData
+ }
+
+ if (!response.data.success && response.data.error) {
+ resp = {
+ ...resp,
+ success: false,
+ error: response.data.error
+ } as ErrorResponse;
+ }
+
+ if (response.data.next) {
+ (resp as CrawlStatusResponse).next = response.data.next;
+ }
+
+ return resp;
+ } else {
+ this.handleError(response, "check crawl status");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Returns information about crawl errors.
+ * @param id - The ID of the crawl operation.
+ * @returns Information about crawl errors.
+ */
+ async checkCrawlErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.deleteRequest(
+ `${this.apiUrl}/v1/crawl/${id}/errors`,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "check crawl errors");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Cancels a crawl job using the Firecrawl API.
+ * @param id - The ID of the crawl operation.
+ * @returns The response from the cancel crawl operation.
+ */
+ async cancelCrawl(id: string): Promise<ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.deleteRequest(
+ `${this.apiUrl}/v1/crawl/${id}`,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "cancel crawl job");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Initiates a crawl job and returns a CrawlWatcher to monitor the job via WebSocket.
+ * @param url - The URL to crawl.
+ * @param params - Additional parameters for the crawl request.
+ * @param idempotencyKey - Optional idempotency key for the request.
+ * @returns A CrawlWatcher instance to monitor the crawl job.
+ */
+ async crawlUrlAndWatch(
+ url: string,
+ params?: CrawlParams,
+ idempotencyKey?: string,
+ ) {
+ const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
+
+ if (crawl.success && crawl.id) {
+ const id = crawl.id;
+ return new CrawlWatcher(id, this);
+ }
+
+ throw new FirecrawlError("Crawl job failed to start", 400);
+ }
+
+ /**
+ * Maps a URL using the Firecrawl API.
+ * @param url - The URL to map.
+ * @param params - Additional parameters for the map request.
+ * @returns The response from the map operation.
+ */
+ async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
+
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/map`,
+ jsonData,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data as MapResponse;
+ } else {
+ this.handleError(response, "map");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Initiates a batch scrape job for multiple URLs using the Firecrawl API.
+ * @param url - The URLs to scrape.
+ * @param params - Additional parameters for the scrape request.
+ * @param pollInterval - Time in seconds for job status checks.
+ * @param idempotencyKey - Optional idempotency key for the request.
+ * @param webhook - Optional webhook for the batch scrape.
+ * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
+ * @returns The response from the crawl operation.
+ */
+ async batchScrapeUrls(
+ urls: string[],
+ params?: ScrapeParams,
+ pollInterval: number = 2,
+ idempotencyKey?: string,
+ webhook?: CrawlParams["webhook"],
+ ignoreInvalidURLs?: boolean,
+ maxConcurrency?: number,
+ ): Promise<BatchScrapeStatusResponse | ErrorResponse> {
+ const headers = this.prepareHeaders(idempotencyKey);
+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: `js-sdk@${this.version}` };
+ if (jsonData?.extract?.schema) {
+ let schema = jsonData.extract.schema;
+
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+
+ }
+ jsonData = {
+ ...jsonData,
+ extract: {
+ ...jsonData.extract,
+ schema: schema,
+ },
+ };
+ }
+ if (jsonData?.jsonOptions?.schema) {
+ let schema = jsonData.jsonOptions.schema;
+
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+
+ }
+ jsonData = {
+ ...jsonData,
+ jsonOptions: {
+ ...jsonData.jsonOptions,
+ schema: schema,
+ },
+ };
+ }
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/batch/scrape`,
+ jsonData,
+ headers
+ );
+ if (response.status === 200) {
+ const id: string = response.data.id;
+ return this.monitorJobStatus(id, headers, pollInterval);
+ } else {
+ this.handleError(response, "start batch scrape job");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ async asyncBatchScrapeUrls(
+ urls: string[],
+ params?: ScrapeParams,
+ idempotencyKey?: string,
+ webhook?: CrawlParams["webhook"],
+ ignoreInvalidURLs?: boolean,
+ ): Promise<BatchScrapeResponse | ErrorResponse> {
+ const headers = this.prepareHeaders(idempotencyKey);
+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/batch/scrape`,
+ jsonData,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start batch scrape job");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.
+ * @param urls - The URL to scrape.
+ * @param params - Additional parameters for the scrape request.
+ * @param idempotencyKey - Optional idempotency key for the request.
+ * @returns A CrawlWatcher instance to monitor the crawl job.
+ */
+ async batchScrapeUrlsAndWatch(
+ urls: string[],
+ params?: ScrapeParams,
+ idempotencyKey?: string,
+ webhook?: CrawlParams["webhook"],
+ ignoreInvalidURLs?: boolean,
+ ) {
+ const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
+
+ if (crawl.success && crawl.id) {
+ const id = crawl.id;
+ return new CrawlWatcher(id, this);
+ }
+
+ throw new FirecrawlError("Batch scrape job failed to start", 400);
+ }
+
+ /**
+ * Checks the status of a batch scrape job using the Firecrawl API.
+ * @param id - The ID of the batch scrape operation.
+ * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
+ * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
+ * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
+ * @param limit - How many entries to return. Only used when `getAllData = false`.
+ * @returns The response containing the job status.
+ */
+ async checkBatchScrapeStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse> {
+ if (!id) {
+ throw new FirecrawlError("No batch scrape ID provided", 400);
+ }
+
+ const headers: AxiosRequestHeaders = this.prepareHeaders();
+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+ if (skip !== undefined) {
+ targetURL.searchParams.set("skip", skip.toString());
+ }
+ if (limit !== undefined) {
+ targetURL.searchParams.set("limit", limit.toString());
+ }
+
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ targetURL.href,
+ headers
+ );
+ if (response.status === 200) {
+ let allData = response.data.data;
+ if (getAllData && response.data.status === "completed") {
+ let statusData = response.data
+ if ("data" in statusData) {
+ let data = statusData.data;
+ while (typeof statusData === 'object' && 'next' in statusData) {
+ if (data.length === 0) {
+ break
+ }
+ statusData = (await this.getRequest(statusData.next, headers)).data;
+ data = data.concat(statusData.data);
+ }
+ allData = data;
+ }
+ }
+
+ let resp: BatchScrapeStatusResponse | ErrorResponse = {
+ success: response.data.success,
+ status: response.data.status,
+ total: response.data.total,
+ completed: response.data.completed,
+ creditsUsed: response.data.creditsUsed,
+ next: getAllData ? undefined : response.data.next,
+ expiresAt: new Date(response.data.expiresAt),
+ data: allData
+ }
+
+ if (!response.data.success && response.data.error) {
+ resp = {
+ ...resp,
+ success: false,
+ error: response.data.error
+ } as ErrorResponse;
+ }
+
+ if (response.data.next) {
+ (resp as BatchScrapeStatusResponse).next = response.data.next;
+ }
+
+ return resp;
+ } else {
+ this.handleError(response, "check batch scrape status");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Returns information about batch scrape errors.
+ * @param id - The ID of the batch scrape operation.
+ * @returns Information about batch scrape errors.
+ */
+ async checkBatchScrapeErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.deleteRequest(
+ `${this.apiUrl}/v1/batch/scrape/${id}/errors`,
+ headers
+ );
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "check batch scrape errors");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Extracts information from URLs using the Firecrawl API.
+ * Currently in Beta. Expect breaking changes on future minor versions.
+ * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
+ * @param params - Additional parameters for the extract request.
+ * @returns The response from the extract operation.
+ */
+ async extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {
+ const headers = this.prepareHeaders();
+
+ let jsonData: { urls?: string[] } & ExtractParams<T> = { urls: urls, ...params };
+ let jsonSchema: any;
+ try {
+ if (!params?.schema) {
+ jsonSchema = undefined;
+ } else {
+ try {
+ jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);
+ } catch (_) {
+ jsonSchema = params.schema;
+ }
+ }
+ } catch (error: any) {
+ throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
+ }
+
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/extract`,
+ { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
+ headers
+ );
+
+ if (response.status === 200) {
+ const jobId = response.data.id;
+ let extractStatus;
+ do {
+ const statusResponse: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/extract/${jobId}`,
+ headers
+ );
+ extractStatus = statusResponse.data;
+ if (extractStatus.status === "completed") {
+ if (extractStatus.success) {
+ return {
+ success: true,
+ data: extractStatus.data,
+ warning: extractStatus.warning,
+ error: extractStatus.error,
+ sources: extractStatus?.sources || undefined,
+ };
+ } else {
+ throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);
+ }
+ } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {
+ throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);
+ }
+ await new Promise(resolve => setTimeout(resolve, 1000)); // Polling interval
+ } while (extractStatus.status !== "completed");
+ } else {
+ this.handleError(response, "extract");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);
+ }
+ return { success: false, error: "Internal server error."};
+ }
+
+ /**
+ * Initiates an asynchronous extract job for a URL using the Firecrawl API.
+ * @param url - The URL to extract data from.
+ * @param params - Additional parameters for the extract request.
+ * @param idempotencyKey - Optional idempotency key for the request.
+ * @returns The response from the extract operation.
+ */
+ async asyncExtract(
+ urls: string[],
+ params?: ExtractParams,
+ idempotencyKey?: string
+ ): Promise<ExtractResponse | ErrorResponse> {
+ const headers = this.prepareHeaders(idempotencyKey);
+ let jsonData: any = { urls, ...params };
+ let jsonSchema: any;
+
+ try {
+ if (!params?.schema) {
+ jsonSchema = undefined;
+ } else {
+ try {
+ jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);
+ } catch (_) {
+ jsonSchema = params.schema;
+ }
+ }
+ } catch (error: any) {
+ throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
+ }
+
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ this.apiUrl + `/v1/extract`,
+ { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start extract job");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Retrieves the status of an extract job.
+ * @param jobId - The ID of the extract job.
+ * @returns The status of the extract job.
+ */
+ async getExtractStatus(jobId: string): Promise<any> {
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/extract/${jobId}`,
+ this.prepareHeaders()
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "get extract status");
+ }
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+
+ /**
+ * Prepares the headers for an API request.
+ * @param idempotencyKey - Optional key to ensure idempotency.
+ * @returns The prepared headers.
+ */
+ prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
+ return {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${this.apiKey}`,
+ ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),
+ } as AxiosRequestHeaders & { "x-idempotency-key"?: string };
+ }
+
+ /**
+ * Sends a POST request to the specified URL.
+ * @param url - The URL to send the request to.
+ * @param data - The data to send in the request.
+ * @param headers - The headers for the request.
+ * @returns The response from the POST request.
+ */
+ postRequest(
+ url: string,
+ data: any,
+ headers: AxiosRequestHeaders
+ ): Promise<AxiosResponse> {
+ return axios.post(url, data, { headers, timeout: (data?.timeout ? (data.timeout + 5000) : undefined) });
+ }
+
+ /**
+ * Sends a GET request to the specified URL.
+ * @param url - The URL to send the request to.
+ * @param headers - The headers for the request.
+ * @returns The response from the GET request.
+ */
+ async getRequest(
+ url: string,
+ headers: AxiosRequestHeaders
+ ): Promise<AxiosResponse> {
+ try {
+ return await axios.get(url, { headers });
+ } catch (error) {
+ if (error instanceof AxiosError && error.response) {
+ return error.response as AxiosResponse;
+ } else {
+ throw error;
+ }
+ }
+ }
+
+ /**
+ * Sends a DELETE request to the specified URL.
+ * @param url - The URL to send the request to.
+ * @param headers - The headers for the request.
+ * @returns The response from the DELETE request.
+ */
+ async deleteRequest(
+ url: string,
+ headers: AxiosRequestHeaders
+ ): Promise<AxiosResponse> {
+ try {
+ return await axios.delete(url, { headers });
+ } catch (error) {
+ if (error instanceof AxiosError && error.response) {
+ return error.response as AxiosResponse;
+ } else {
+ throw error;
+ }
+ }
+ }
+
+ /**
+ * Monitors the status of a crawl job until completion or failure.
+ * @param id - The ID of the crawl operation.
+ * @param headers - The headers for the request.
+ * @param checkInterval - Interval in seconds for job status checks.
+ * @param checkUrl - Optional URL to check the status (used for v1 API)
+ * @returns The final job status or data.
+ */
+ async monitorJobStatus(
+ id: string,
+ headers: AxiosRequestHeaders,
+ checkInterval: number
+ ): Promise<CrawlStatusResponse | ErrorResponse> {
+ let failedTries = 0;
+ let networkRetries = 0;
+ const maxNetworkRetries = 3;
+
+ while (true) {
+ try {
+ let statusResponse: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/crawl/${id}`,
+ headers
+ );
+
+ if (statusResponse.status === 200) {
+ failedTries = 0;
+ networkRetries = 0;
+ let statusData = statusResponse.data;
+
+ if (statusData.status === "completed") {
+ if ("data" in statusData) {
+ let data = statusData.data;
+ while (typeof statusData === 'object' && 'next' in statusData) {
+ if (data.length === 0) {
+ break
+ }
+ statusResponse = await this.getRequest(statusData.next, headers);
+ statusData = statusResponse.data;
+ data = data.concat(statusData.data);
+ }
+ statusData.data = data;
+ return statusData;
+ } else {
+ throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+ }
+ } else if (
+ ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
+ ) {
+ checkInterval = Math.max(checkInterval, 2);
+ await new Promise((resolve) =>
+ setTimeout(resolve, checkInterval * 1000)
+ );
+ } else {
+ throw new FirecrawlError(
+ `Crawl job failed or was stopped. Status: ${statusData.status}`,
+ 500
+ );
+ }
+ } else {
+ failedTries++;
+ if (failedTries >= 3) {
+ this.handleError(statusResponse, "check crawl status");
+ }
+ }
+ } catch (error: any) {
+ if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {
+ networkRetries++;
+ const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);
+
+ await new Promise((resolve) => setTimeout(resolve, backoffDelay));
+ continue;
+ }
+
+ throw new FirecrawlError(error, 500);
+ }
+ }
+ }
+
+ /**
+ * Determines if an error is retryable (transient network error)
+ * @param error - The error to check
+ * @returns True if the error should be retried
+ */
+ private isRetryableError(error: any): boolean {
+ if (error instanceof AxiosError) {
+ if (!error.response) {
+ const code = error.code;
+ const message = error.message?.toLowerCase() || '';
+
+ return (
+ code === 'ECONNRESET' ||
+ code === 'ETIMEDOUT' ||
+ code === 'ENOTFOUND' ||
+ code === 'ECONNREFUSED' ||
+ message.includes('socket hang up') ||
+ message.includes('network error') ||
+ message.includes('timeout')
+ );
+ }
+
+ if (error.response?.status === 408 || error.response?.status === 504) {
+ return true;
+ }
+ }
+
+ if (error && typeof error === 'object') {
+ const code = error.code;
+ const message = error.message?.toLowerCase() || '';
+
+ if (code === 'ECONNRESET' ||
+ code === 'ETIMEDOUT' ||
+ code === 'ENOTFOUND' ||
+ code === 'ECONNREFUSED' ||
+ message.includes('socket hang up') ||
+ message.includes('network error') ||
+ message.includes('timeout')) {
+ return true;
+ }
+
+ if (error.response?.status === 408 || error.response?.status === 504) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Handles errors from API responses.
+ * @param {AxiosResponse} response - The response from the API.
+ * @param {string} action - The action being performed when the error occurred.
+ */
+ handleError(response: AxiosResponse, action: string): void {
+ if (!response) {
+ throw new FirecrawlError(
+ `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,
+ 0
+ );
+ }
+
+ if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
+ const errorMessage: string =
+ response.data.error || "Unknown error occurred";
+ const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : '';
+ throw new FirecrawlError(
+ `Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}${details}`,
+ response.status,
+ response?.data?.details
+ );
+ } else {
+ throw new FirecrawlError(
+ `Unexpected error occurred while trying to ${action}. Status code: ${response.status}`,
+ response.status
+ );
+ }
+ }
+
+ /**
+ * Initiates a deep research operation on a given query and polls until completion.
+ * @param query - The query to research.
+ * @param params - Parameters for the deep research operation.
+ * @param onActivity - Optional callback to receive activity updates in real-time.
+ * @param onSource - Optional callback to receive source updates in real-time.
+ * @returns The final research results.
+ */
+ async deepResearch(
+ query: string,
+ params: DeepResearchParams<zt.ZodSchema>,
+ onActivity?: (activity: {
+ type: string;
+ status: string;
+ message: string;
+ timestamp: string;
+ depth: number;
+ }) => void,
+ onSource?: (source: {
+ url: string;
+ title?: string;
+ description?: string;
+ icon?: string;
+ }) => void
+ ): Promise<DeepResearchStatusResponse | ErrorResponse> {
+ try {
+ const response = await this.asyncDeepResearch(query, params);
+
+ if (!response.success || 'error' in response) {
+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };
+ }
+
+ if (!response.id) {
+ throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);
+ }
+
+ const jobId = response.id;
+ let researchStatus;
+ let lastActivityCount = 0;
+ let lastSourceCount = 0;
+
+ while (true) {
+ researchStatus = await this.checkDeepResearchStatus(jobId);
+
+ if ('error' in researchStatus && !researchStatus.success) {
+ return researchStatus;
+ }
+
+ // Stream new activities through the callback if provided
+ if (onActivity && researchStatus.activities) {
+ const newActivities = researchStatus.activities.slice(lastActivityCount);
+ for (const activity of newActivities) {
+ onActivity(activity);
+ }
+ lastActivityCount = researchStatus.activities.length;
+ }
+
+ // Stream new sources through the callback if provided
+ if (onSource && researchStatus.sources) {
+ const newSources = researchStatus.sources.slice(lastSourceCount);
+ for (const source of newSources) {
+ onSource(source);
+ }
+ lastSourceCount = researchStatus.sources.length;
+ }
+
+ if (researchStatus.status === "completed") {
+ return researchStatus;
+ }
+
+ if (researchStatus.status === "failed") {
+ throw new FirecrawlError(
+ `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,
+ 500
+ );
+ }
+
+ if (researchStatus.status !== "processing") {
+ break;
+ }
+
+ await new Promise(resolve => setTimeout(resolve, 2000));
+ }
+
+ return { success: false, error: "Research job terminated unexpectedly" };
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);
+ }
+ }
+
+ /**
+ * Initiates a deep research operation on a given query without polling.
+ * @param params - Parameters for the deep research operation.
+ * @returns The response containing the research job ID.
+ */
+ async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ let jsonData: any = { query, ...params, origin: `js-sdk@${this.version}` };
+
+ if (jsonData?.jsonOptions?.schema) {
+ let schema = jsonData.jsonOptions.schema;
+ // Try parsing the schema as a Zod schema
+ try {
+ schema = zodToJsonSchema(schema);
+ } catch (error) {
+ // Ignore error if schema can't be parsed as Zod
+ }
+ jsonData = {
+ ...jsonData,
+ jsonOptions: {
+ ...jsonData.jsonOptions,
+ schema: schema,
+ },
+ };
+ }
+
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ `${this.apiUrl}/v1/deep-research`,
+ jsonData,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start deep research");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Checks the status of a deep research operation.
+ * @param id - The ID of the deep research operation.
+ * @returns The current status and results of the research operation.
+ */
+ async checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/deep-research/${id}`,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else if (response.status === 404) {
+ throw new FirecrawlError("Deep research job not found", 404);
+ } else {
+ this.handleError(response, "check deep research status");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * @deprecated Use deepResearch() instead
+ * Initiates a deep research operation on a given topic and polls until completion.
+ * @param topic - The topic to research.
+ * @param params - Parameters for the deep research operation.
+ * @param onActivity - Optional callback to receive activity updates in real-time.
+ * @returns The final research results.
+ */
+ async __deepResearch(
+ topic: string,
+ params: DeepResearchParams,
+ onActivity?: (activity: {
+ type: string;
+ status: string;
+ message: string;
+ timestamp: string;
+ depth: number;
+ }) => void
+ ): Promise<DeepResearchStatusResponse | ErrorResponse> {
+ try {
+ const response = await this.__asyncDeepResearch(topic, params);
+
+ if (!response.success || 'error' in response) {
+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };
+ }
+
+ if (!response.id) {
+ throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);
+ }
+
+ const jobId = response.id;
+ let researchStatus;
+ let lastActivityCount = 0;
+
+ while (true) {
+ researchStatus = await this.__checkDeepResearchStatus(jobId);
+
+ if ('error' in researchStatus && !researchStatus.success) {
+ return researchStatus;
+ }
+
+ // Stream new activities through the callback if provided
+ if (onActivity && researchStatus.activities) {
+ const newActivities = researchStatus.activities.slice(lastActivityCount);
+ for (const activity of newActivities) {
+ onActivity(activity);
+ }
+ lastActivityCount = researchStatus.activities.length;
+ }
+
+ if (researchStatus.status === "completed") {
+ return researchStatus;
+ }
+
+ if (researchStatus.status === "failed") {
+ throw new FirecrawlError(
+ `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,
+ 500
+ );
+ }
+
+ if (researchStatus.status !== "processing") {
+ break;
+ }
+
+ await new Promise(resolve => setTimeout(resolve, 2000));
+ }
+
+ return { success: false, error: "Research job terminated unexpectedly" };
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);
+ }
+ }
+
+ /**
+ * @deprecated Use asyncDeepResearch() instead
+ * Initiates a deep research operation on a given topic without polling.
+ * @param params - Parameters for the deep research operation.
+ * @returns The response containing the research job ID.
+ */
+ async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ let jsonData: any = { topic, ...params, origin: `js-sdk@${this.version}` };
+ const response: AxiosResponse = await this.postRequest(
+ `${this.apiUrl}/v1/deep-research`,
+ jsonData,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start deep research");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * @deprecated Use checkDeepResearchStatus() instead
+ * Checks the status of a deep research operation.
+ * @param id - The ID of the deep research operation.
+ * @returns The current status and results of the research operation.
+ */
+ async __checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/deep-research/${id}`,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else if (response.status === 404) {
+ throw new FirecrawlError("Deep research job not found", 404);
+ } else {
+ this.handleError(response, "check deep research status");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Generates LLMs.txt for a given URL and polls until completion.
+ * @param url - The URL to generate LLMs.txt from.
+ * @param params - Parameters for the LLMs.txt generation operation.
+ * @returns The final generation results.
+ */
+ async generateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {
+ try {
+ const response = await this.asyncGenerateLLMsText(url, params);
+
+ if (!response.success || 'error' in response) {
+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };
+ }
+
+ if (!response.id) {
+ throw new FirecrawlError(`Failed to start LLMs.txt generation. No job ID returned.`, 500);
+ }
+
+ const jobId = response.id;
+ let generationStatus;
+
+ while (true) {
+ generationStatus = await this.checkGenerateLLMsTextStatus(jobId);
+
+ if ('error' in generationStatus && !generationStatus.success) {
+ return generationStatus;
+ }
+
+ if (generationStatus.status === "completed") {
+ return generationStatus;
+ }
+
+ if (generationStatus.status === "failed") {
+ throw new FirecrawlError(
+ `LLMs.txt generation job ${generationStatus.status}. Error: ${generationStatus.error}`,
+ 500
+ );
+ }
+
+ if (generationStatus.status !== "processing") {
+ break;
+ }
+
+ await new Promise(resolve => setTimeout(resolve, 2000));
+ }
+
+ return { success: false, error: "LLMs.txt generation job terminated unexpectedly" };
+ } catch (error: any) {
+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);
+ }
+ }
+
+ /**
+ * Initiates a LLMs.txt generation operation without polling.
+ * @param url - The URL to generate LLMs.txt from.
+ * @param params - Parameters for the LLMs.txt generation operation.
+ * @returns The response containing the generation job ID.
+ */
+ async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };
+ try {
+ const response: AxiosResponse = await this.postRequest(
+ `${this.apiUrl}/v1/llmstxt`,
+ jsonData,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "start LLMs.txt generation");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+
+ /**
+ * Checks the status of a LLMs.txt generation operation.
+ * @param id - The ID of the LLMs.txt generation operation.
+ * @returns The current status and results of the generation operation.
+ */
+ async checkGenerateLLMsTextStatus(id: string): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/llmstxt/${id}`,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else if (response.status === 404) {
+ throw new FirecrawlError("LLMs.txt generation job not found", 404);
+ } else {
+ this.handleError(response, "check LLMs.txt generation status");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
+}
+
+interface CrawlWatcherEvents {
+ document: CustomEvent<FirecrawlDocument<undefined>>,
+ done: CustomEvent<{
+ status: CrawlStatusResponse["status"];
+ data: FirecrawlDocument<undefined>[];
+ }>,
+ error: CustomEvent<{
+ status: CrawlStatusResponse["status"],
+ data: FirecrawlDocument<undefined>[],
+ error: string,
+ }>,
+}
+
+export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
+ private ws: WebSocket;
+ public data: FirecrawlDocument<undefined>[];
+ public status: CrawlStatusResponse["status"];
+ public id: string;
+
+ constructor(id: string, app: FirecrawlApp) {
+ super();
+ this.id = id;
+ // replace `http` with `ws` (`http://` -> `ws://` and `https://` -> `wss://`)
+ const wsUrl = app.apiUrl.replace(/^http/, "ws");
+ this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
+ this.status = "scraping";
+ this.data = [];
+
+ type ErrorMessage = {
+ type: "error",
+ error: string,
+ }
+
+ type CatchupMessage = {
+ type: "catchup",
+ data: CrawlStatusResponse,
+ }
+
+ type DocumentMessage = {
+ type: "document",
+ data: FirecrawlDocument<undefined>,
+ }
+
+ type DoneMessage = { type: "done" }
+
+ type Message = ErrorMessage | CatchupMessage | DoneMessage | DocumentMessage;
+
+ const messageHandler = (msg: Message) => {
+ if (msg.type === "done") {
+ this.status = "completed";
+ this.dispatchTypedEvent("done", new CustomEvent("done", {
+ detail: {
+ status: this.status,
+ data: this.data,
+ id: this.id,
+ },
+ }));
+ } else if (msg.type === "error") {
+ this.status = "failed";
+ this.dispatchTypedEvent("error", new CustomEvent("error", {
+ detail: {
+ status: this.status,
+ data: this.data,
+ error: msg.error,
+ id: this.id,
+ },
+ }));
+ } else if (msg.type === "catchup") {
+ this.status = msg.data.status;
+ this.data.push(...(msg.data.data ?? []));
+ for (const doc of this.data) {
+ this.dispatchTypedEvent("document", new CustomEvent("document", {
+ detail: {
+ ...doc,
+ id: this.id,
+ },
+ }));
+ }
+ } else if (msg.type === "document") {
+ this.dispatchTypedEvent("document", new CustomEvent("document", {
+ detail: {
+ ...msg.data,
+ id: this.id,
+ },
+ }));
+ }
+ }
+
+ this.ws.onmessage = ((ev: MessageEvent) => {
+ if (typeof ev.data !== "string") {
+ this.ws.close();
+ return;
+ }
+ try {
+ const msg = JSON.parse(ev.data) as Message;
+ messageHandler(msg);
+ } catch (error) {
+ console.error("Error on message", error);
+ }
+ }).bind(this);
+
+ this.ws.onclose = ((ev: CloseEvent) => {
+ try {
+ const msg = JSON.parse(ev.reason) as Message;
+ messageHandler(msg);
+ } catch (error) {
+ console.error("Error on close", error);
+ }
+ }).bind(this);
+
+ this.ws.onerror = ((_: Event) => {
+ this.status = "failed"
+ this.dispatchTypedEvent("error", new CustomEvent("error", {
+ detail: {
+ status: this.status,
+ data: this.data,
+ error: "WebSocket error",
+ id: this.id,
+ },
+ }));
+ }).bind(this);
+ }
+
+ close() {
+ this.ws.close();
+ }
+}
diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.ts
new file mode 100644
index 000000000..8987f6de5
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/client.ts
@@ -0,0 +1,151 @@
+import { HttpClient } from "./utils/httpClient";
+import { scrape } from "./methods/scrape";
+import { search } from "./methods/search";
+import { map as mapMethod } from "./methods/map";
+import {
+ startCrawl,
+ getCrawlStatus,
+ cancelCrawl,
+ crawl as crawlWaiter,
+ getCrawlErrors,
+ getActiveCrawls,
+ crawlParamsPreview,
+} from "./methods/crawl";
+import {
+ startBatchScrape,
+ getBatchScrapeStatus,
+ getBatchScrapeErrors,
+ cancelBatchScrape,
+ batchScrape as batchWaiter,
+} from "./methods/batch";
+import { startExtract, getExtractStatus, extract as extractWaiter } from "./methods/extract";
+import { getConcurrency, getCreditUsage, getTokenUsage } from "./methods/usage";
+import type {
+ Document,
+ ScrapeOptions,
+ SearchData,
+ SearchRequest,
+ MapData,
+ MapOptions,
+ CrawlResponse,
+ CrawlJob,
+ CrawlErrorsResponse,
+ ActiveCrawlsResponse,
+ BatchScrapeResponse,
+ BatchScrapeJob,
+ ExtractResponse,
+} from "./types";
+import { Watcher } from "./watcher";
+import type { WatcherOptions } from "./watcher";
+
+export interface FirecrawlClientOptions {
+ apiKey?: string | null;
+ apiUrl?: string | null;
+ timeoutMs?: number;
+ maxRetries?: number;
+ backoffFactor?: number;
+}
+
+export class FirecrawlClient {
+ private readonly http: HttpClient;
+
+ constructor(options: FirecrawlClientOptions = {}) {
+ const apiKey = options.apiKey ?? process.env.FIRECRAWL_API_KEY ?? "";
+ const apiUrl = (options.apiUrl ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev").replace(/\/$/, "");
+ if (!apiKey) {
+ throw new Error("API key is required. Set FIRECRAWL_API_KEY env or pass apiKey.");
+ }
+ this.http = new HttpClient({
+ apiKey,
+ apiUrl,
+ timeoutMs: options.timeoutMs,
+ maxRetries: options.maxRetries,
+ backoffFactor: options.backoffFactor,
+ });
+ }
+
+ // Scrape
+ async scrape(url: string, options?: ScrapeOptions): Promise<Document> {
+ return scrape(this.http, url, options);
+ }
+
+ // Search
+ async search(query: string, req: Omit<SearchRequest, "query"> = {}): Promise<SearchData> {
+ return search(this.http, { query, ...req });
+ }
+
+ // Map
+ async map(url: string, options?: MapOptions): Promise<MapData> {
+ return mapMethod(this.http, url, options);
+ }
+
+ // Crawl
+ async startCrawl(url: string, req: Omit<Parameters<typeof startCrawl>[1], "url"> = {}): Promise<CrawlResponse> {
+ return startCrawl(this.http, { url, ...(req as any) });
+ }
+ async getCrawlStatus(jobId: string): Promise<CrawlJob> {
+ return getCrawlStatus(this.http, jobId);
+ }
+ async cancelCrawl(jobId: string): Promise<boolean> {
+ return cancelCrawl(this.http, jobId);
+ }
+ async crawl(url: string, req: Omit<Parameters<typeof startCrawl>[1], "url"> & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
+ return crawlWaiter(this.http, { url, ...(req as any) }, req.pollInterval, req.timeout);
+ }
+ async getCrawlErrors(crawlId: string): Promise<CrawlErrorsResponse> {
+ return getCrawlErrors(this.http, crawlId);
+ }
+ async getActiveCrawls(): Promise<ActiveCrawlsResponse> {
+ return getActiveCrawls(this.http);
+ }
+ async crawlParamsPreview(url: string, prompt: string): Promise<Record<string, unknown>> {
+ return crawlParamsPreview(this.http, url, prompt);
+ }
+
+ // Batch
+ async startBatchScrape(urls: string[], opts?: Parameters<typeof startBatchScrape>[2]): Promise<BatchScrapeResponse> {
+ return startBatchScrape(this.http, urls, opts);
+ }
+ async getBatchScrapeStatus(jobId: string): Promise<BatchScrapeJob> {
+ return getBatchScrapeStatus(this.http, jobId);
+ }
+ async getBatchScrapeErrors(jobId: string): Promise<CrawlErrorsResponse> {
+ return getBatchScrapeErrors(this.http, jobId);
+ }
+ async cancelBatchScrape(jobId: string): Promise<boolean> {
+ return cancelBatchScrape(this.http, jobId);
+ }
+ async batchScrape(urls: string[], opts?: Parameters<typeof startBatchScrape>[2] & { pollInterval?: number; timeout?: number }): Promise<BatchScrapeJob> {
+ return batchWaiter(this.http, urls, opts);
+ }
+
+ // Extract
+ async startExtract(args: Parameters<typeof startExtract>[1]): Promise<ExtractResponse> {
+ return startExtract(this.http, args);
+ }
+ async getExtractStatus(jobId: string): Promise<ExtractResponse> {
+ return getExtractStatus(this.http, jobId);
+ }
+ async extract(args: Parameters<typeof startExtract>[1] & { pollInterval?: number; timeout?: number }): Promise<ExtractResponse> {
+ return extractWaiter(this.http, args);
+ }
+
+ // Usage
+ async getConcurrency() {
+ return getConcurrency(this.http);
+ }
+ async getCreditUsage() {
+ return getCreditUsage(this.http);
+ }
+ async getTokenUsage() {
+ return getTokenUsage(this.http);
+ }
+
+ // Watcher
+ watcher(jobId: string, opts: WatcherOptions = {}): Watcher {
+ return new Watcher(this.http, jobId, opts);
+ }
+}
+
+export default FirecrawlClient;
+
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
new file mode 100644
index 000000000..796dd54c4
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts
@@ -0,0 +1,131 @@
+import {
+ type BatchScrapeJob,
+ type BatchScrapeResponse,
+ type CrawlErrorsResponse,
+ type Document,
+ type ScrapeOptions,
+ type WebhookConfig,
+} from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { ensureValidScrapeOptions } from "../utils/validation";
+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+
+export interface StartBatchOptions {
+ options?: ScrapeOptions;
+ webhook?: string | WebhookConfig;
+ appendToId?: string;
+ ignoreInvalidURLs?: boolean;
+ maxConcurrency?: number;
+ zeroDataRetention?: boolean;
+ integration?: string;
+ idempotencyKey?: string;
+}
+
+export async function startBatchScrape(
+ http: HttpClient,
+ urls: string[],
+ {
+ options,
+ webhook,
+ appendToId,
+ ignoreInvalidURLs,
+ maxConcurrency,
+ zeroDataRetention,
+ integration,
+ idempotencyKey,
+ }: StartBatchOptions = {}
+): Promise<BatchScrapeResponse> {
+ if (!Array.isArray(urls) || urls.length === 0) throw new Error("URLs list cannot be empty");
+ const payload: Record<string, unknown> = { urls };
+ if (options) {
+ ensureValidScrapeOptions(options);
+ Object.assign(payload, options);
+ }
+ if (webhook != null) payload.webhook = webhook;
+ if (appendToId != null) payload.appendToId = appendToId;
+ if (ignoreInvalidURLs != null) payload.ignoreInvalidURLs = ignoreInvalidURLs;
+ if (maxConcurrency != null) payload.maxConcurrency = maxConcurrency;
+ if (zeroDataRetention != null) payload.zeroDataRetention = zeroDataRetention;
+ if (integration != null) payload.integration = integration;
+
+ try {
+ const headers = http.prepareHeaders(idempotencyKey);
+ const res = await http.post<{ success: boolean; id: string; url: string; invalidURLs?: string[]; error?: string }>("/v2/batch/scrape", payload, headers);
+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "start batch scrape");
+ return { id: res.data.id, url: res.data.url, invalidURLs: res.data.invalidURLs || undefined };
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "start batch scrape");
+ throw err;
+ }
+}
+
+export async function getBatchScrapeStatus(http: HttpClient, jobId: string): Promise<BatchScrapeJob> {
+ try {
+ const res = await http.get<{ success: boolean; status: BatchScrapeJob["status"]; completed?: number; total?: number; creditsUsed?: number; expiresAt?: string; next?: string | null; data?: Document[] }>(`/v2/batch/scrape/${jobId}`);
+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get batch scrape status");
+ const body = res.data;
+ return {
+ status: body.status,
+ completed: body.completed ?? 0,
+ total: body.total ?? 0,
+ creditsUsed: body.creditsUsed,
+ expiresAt: body.expiresAt,
+ next: body.next ?? null,
+ data: (body.data || []) as Document[],
+ };
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "get batch scrape status");
+ throw err;
+ }
+}
+
+export async function cancelBatchScrape(http: HttpClient, jobId: string): Promise<boolean> {
+ try {
+ const res = await http.delete<{ status: string }>(`/v2/batch/scrape/${jobId}`);
+ if (res.status !== 200) throwForBadResponse(res, "cancel batch scrape");
+ return res.data?.status === "cancelled";
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "cancel batch scrape");
+ throw err;
+ }
+}
+
+export async function getBatchScrapeErrors(http: HttpClient, jobId: string): Promise<CrawlErrorsResponse> {
+ try {
+ const res = await http.get<{ success?: boolean; data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] } }>(`/v2/batch/scrape/${jobId}/errors`);
+ if (res.status !== 200) throwForBadResponse(res, "get batch scrape errors");
+ const payload = res.data?.data ?? (res.data as any);
+ return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "get batch scrape errors");
+ throw err;
+ }
+}
+
+export async function waitForBatchCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<BatchScrapeJob> {
+ const start = Date.now();
+ while (true) {
+ const status = await getBatchScrapeStatus(http, jobId);
+ if (["completed", "failed", "cancelled"].includes(status.status)) return status;
+ if (timeout != null && Date.now() - start > timeout * 1000) {
+ throw new Error(`Batch scrape job ${jobId} did not complete within ${timeout} seconds`);
+ }
+ await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
+ }
+}
+
+export async function batchScrape(
+ http: HttpClient,
+ urls: string[],
+ opts: StartBatchOptions & { pollInterval?: number; timeout?: number } = {}
+): Promise<BatchScrapeJob> {
+ const start = await startBatchScrape(http, urls, opts);
+ return waitForBatchCompletion(http, start.id, opts.pollInterval ?? 2, opts.timeout);
+}
+
+export function chunkUrls(urls: string[], chunkSize = 100): string[][] {
+ const chunks: string[][] = [];
+ for (let i = 0; i < urls.length; i += chunkSize) chunks.push(urls.slice(i, i + chunkSize));
+ return chunks;
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
new file mode 100644
index 000000000..74a6bfd4c
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
@@ -0,0 +1,160 @@
+import {
+ type ActiveCrawlsResponse,
+ type CrawlErrorsResponse,
+ type CrawlJob,
+ type CrawlResponse,
+ type Document,
+ type ScrapeOptions,
+ type WebhookConfig,
+} from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { ensureValidScrapeOptions } from "../utils/validation";
+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+
+export interface CrawlRequest {
+ url: string;
+ prompt?: string | null;
+ excludePaths?: string[] | null;
+ includePaths?: string[] | null;
+ maxDiscoveryDepth?: number | null;
+ ignoreSitemap?: boolean;
+ ignoreQueryParameters?: boolean;
+ limit?: number | null;
+ crawlEntireDomain?: boolean;
+ allowExternalLinks?: boolean;
+ allowSubdomains?: boolean;
+ delay?: number | null;
+ maxConcurrency?: number | null;
+ webhook?: string | WebhookConfig | null;
+ scrapeOptions?: ScrapeOptions | null;
+ zeroDataRetention?: boolean;
+}
+
+function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {
+ if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");
+ const data: Record<string, unknown> = { url: request.url.trim() };
+ if (request.prompt) data.prompt = request.prompt;
+ if (request.excludePaths) data.excludePaths = request.excludePaths;
+ if (request.includePaths) data.includePaths = request.includePaths;
+ if (request.maxDiscoveryDepth != null) data.maxDiscoveryDepth = request.maxDiscoveryDepth;
+ if (request.ignoreSitemap != null) data.ignoreSitemap = request.ignoreSitemap;
+ if (request.ignoreQueryParameters != null) data.ignoreQueryParameters = request.ignoreQueryParameters;
+ if (request.limit != null) data.limit = request.limit;
+ if (request.crawlEntireDomain != null) data.crawlEntireDomain = request.crawlEntireDomain;
+ if (request.allowExternalLinks != null) data.allowExternalLinks = request.allowExternalLinks;
+ if (request.allowSubdomains != null) data.allowSubdomains = request.allowSubdomains;
+ if (request.delay != null) data.delay = request.delay;
+ if (request.maxConcurrency != null) data.maxConcurrency = request.maxConcurrency;
+ if (request.webhook != null) data.webhook = request.webhook;
+ if (request.scrapeOptions) {
+ ensureValidScrapeOptions(request.scrapeOptions);
+ data.scrapeOptions = request.scrapeOptions;
+ }
+ if (request.zeroDataRetention != null) data.zeroDataRetention = request.zeroDataRetention;
+ return data;
+}
+
+export async function startCrawl(http: HttpClient, request: CrawlRequest): Promise<CrawlResponse> {
+ const payload = prepareCrawlPayload(request);
+ try {
+ const res = await http.post<{ success: boolean; id: string; url: string; error?: string }>("/v2/crawl", payload);
+ if (res.status !== 200 || !res.data?.success) {
+ throwForBadResponse(res, "start crawl");
+ }
+ return { id: res.data.id, url: res.data.url };
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "start crawl");
+ throw err;
+ }
+}
+
+export async function getCrawlStatus(http: HttpClient, jobId: string): Promise<CrawlJob> {
+ try {
+ const res = await http.get<{ success: boolean; status: CrawlJob["status"]; completed?: number; total?: number; creditsUsed?: number; expiresAt?: string; next?: string | null; data?: Document[] }>(`/v2/crawl/${jobId}`);
+ if (res.status !== 200 || !res.data?.success) {
+ throwForBadResponse(res, "get crawl status");
+ }
+ const body = res.data;
+ return {
+ status: body.status,
+ completed: body.completed ?? 0,
+ total: body.total ?? 0,
+ creditsUsed: body.creditsUsed,
+ expiresAt: body.expiresAt,
+ next: body.next ?? null,
+ data: (body.data || []) as Document[],
+ };
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "get crawl status");
+ throw err;
+ }
+}
+
+export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolean> {
+ try {
+ const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`);
+ if (res.status !== 200) throwForBadResponse(res, "cancel crawl");
+ return res.data?.status === "cancelled";
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");
+ throw err;
+ }
+}
+
+export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
+ const start = Date.now();
+ while (true) {
+ const status = await getCrawlStatus(http, jobId);
+ if (["completed", "failed", "cancelled"].includes(status.status)) return status;
+ if (timeout != null && Date.now() - start > timeout * 1000) {
+ throw new Error(`Crawl job ${jobId} did not complete within ${timeout} seconds`);
+ }
+ await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
+ }
+}
+
+export async function crawl(http: HttpClient, request: CrawlRequest, pollInterval = 2, timeout?: number): Promise<CrawlJob> {
+ const started = await startCrawl(http, request);
+ return waitForCrawlCompletion(http, started.id, pollInterval, timeout);
+}
+
+export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {
+ try {
+ const res = await http.get<{ success?: boolean; data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] } }>(`/v2/crawl/${crawlId}/errors`);
+ if (res.status !== 200) throwForBadResponse(res, "get crawl errors");
+ const payload = res.data?.data ?? (res.data as any);
+ return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "get crawl errors");
+ throw err;
+ }
+}
+
+export async function getActiveCrawls(http: HttpClient): Promise<ActiveCrawlsResponse> {
+ try {
+ const res = await http.get<{ success: boolean; data?: { crawls: Array<{ id: string; teamId?: string; team_id?: string; url: string; options?: any }> } }>(`/v2/crawl/active`);
+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get active crawls");
+ const crawlsIn = res.data?.data?.crawls || [];
+ const crawls = crawlsIn.map((c) => ({ id: c.id, teamId: (c as any).teamId ?? (c as any).team_id, url: c.url, options: c.options ?? null }));
+ return { success: true, crawls };
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "get active crawls");
+ throw err;
+ }
+}
+
+export async function crawlParamsPreview(http: HttpClient, url: string, prompt: string): Promise<Record<string, unknown>> {
+ if (!url || !url.trim()) throw new Error("URL cannot be empty");
+ if (!prompt || !prompt.trim()) throw new Error("Prompt cannot be empty");
+ try {
+ const res = await http.post<{ success: boolean; data?: Record<string, unknown>; warning?: string }>("/v2/crawl/params-preview", { url: url.trim(), prompt });
+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "crawl params preview");
+ const data = res.data.data || {};
+ if (res.data.warning) (data as any).warning = res.data.warning;
+ return data;
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "crawl params preview");
+ throw err;
+ }
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/extract.ts b/apps/js-sdk/firecrawl/src/v2/methods/extract.ts
new file mode 100644
index 000000000..ebcd2f52d
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/methods/extract.ts
@@ -0,0 +1,80 @@
+import { type ExtractResponse, type ScrapeOptions } from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { ensureValidScrapeOptions } from "../utils/validation";
+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+
+function prepareExtractPayload(args: {
+ urls?: string[];
+ prompt?: string;
+ schema?: Record<string, unknown>;
+ systemPrompt?: string;
+ allowExternalLinks?: boolean;
+ enableWebSearch?: boolean;
+ showSources?: boolean;
+ scrapeOptions?: ScrapeOptions;
+ ignoreInvalidURLs?: boolean;
+}): Record<string, unknown> {
+ const body: Record<string, unknown> = {};
+ if (args.urls) body.urls = args.urls;
+ if (args.prompt != null) body.prompt = args.prompt;
+ if (args.schema != null) body.schema = args.schema;
+ if (args.systemPrompt != null) body.systemPrompt = args.systemPrompt;
+ if (args.allowExternalLinks != null) body.allowExternalLinks = args.allowExternalLinks;
+ if (args.enableWebSearch != null) body.enableWebSearch = args.enableWebSearch;
+ if (args.showSources != null) body.showSources = args.showSources;
+ if (args.ignoreInvalidURLs != null) body.ignoreInvalidURLs = args.ignoreInvalidURLs;
+ if (args.scrapeOptions) {
+ ensureValidScrapeOptions(args.scrapeOptions);
+ body.scrapeOptions = args.scrapeOptions;
+ }
+ return body;
+}
+
+export async function startExtract(http: HttpClient, args: Parameters<typeof prepareExtractPayload>[0]): Promise<ExtractResponse> {
+ const payload = prepareExtractPayload(args);
+ try {
+ const res = await http.post<ExtractResponse>("/v2/extract", payload);
+ if (res.status !== 200) throwForBadResponse(res, "extract");
+ return res.data;
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "extract");
+ throw err;
+ }
+}
+
+export async function getExtractStatus(http: HttpClient, jobId: string): Promise<ExtractResponse> {
+ try {
+ const res = await http.get<ExtractResponse>(`/v2/extract/${jobId}`);
+ if (res.status !== 200) throwForBadResponse(res, "extract status");
+ return res.data;
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "extract status");
+ throw err;
+ }
+}
+
+export async function waitExtract(
+ http: HttpClient,
+ jobId: string,
+ pollInterval = 2,
+ timeout?: number
+): Promise<ExtractResponse> {
+ const start = Date.now();
+ while (true) {
+ const status = await getExtractStatus(http, jobId);
+ if (["completed", "failed", "cancelled"].includes(status.status || "")) return status;
+ if (timeout != null && Date.now() - start > timeout * 1000) return status;
+ await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));
+ }
+}
+
+export async function extract(
+ http: HttpClient,
+ args: Parameters<typeof prepareExtractPayload>[0] & { pollInterval?: number; timeout?: number }
+): Promise<ExtractResponse> {
+ const started = await startExtract(http, args);
+ const jobId = started.id;
+ if (!jobId) return started;
+ return waitExtract(http, jobId, args.pollInterval ?? 2, args.timeout);
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/map.ts b/apps/js-sdk/firecrawl/src/v2/methods/map.ts
new file mode 100644
index 000000000..8b41f772e
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/methods/map.ts
@@ -0,0 +1,37 @@
+import { type MapData, type MapOptions, type SearchResult } from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";
+
+function prepareMapPayload(url: string, options?: MapOptions): Record<string, unknown> {
+ if (!url || !url.trim()) throw new Error("URL cannot be empty");
+ const payload: Record<string, unknown> = { url: url.trim() };
+ if (options) {
+ if (options.sitemap != null) payload.sitemap = options.sitemap;
+ if (options.search != null) payload.search = options.search;
+ if (options.includeSubdomains != null) payload.includeSubdomains = options.includeSubdomains;
+ if (options.limit != null) payload.limit = options.limit;
+ if (options.timeout != null) payload.timeout = options.timeout;
+ }
+ return payload;
+}
+
+export async function map(http: HttpClient, url: string, options?: MapOptions): Promise<MapData> {
+ const payload = prepareMapPayload(url, options);
+ try {
+ const res = await http.post<{ success: boolean; error?: string; links?: Array<string | SearchResult> }>("/v2/map", payload);
+ if (res.status !== 200 || !res.data?.success) {
+ throwForBadResponse(res, "map");
+ }
+ const linksIn = res.data.links || [];
+ const links: SearchResult[] = [];
+ for (const item of linksIn) {
+ if (typeof item === "string") links.push({ url: item });
+ else if (item && typeof item === "object") links.push({ url: item.url, title: (item as any).title, description: (item as any).description });
+ }
+ return { links };
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "map");
+ throw err;
+ }
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts b/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts
new file mode 100644
index 000000000..c2ddfc045
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts
@@ -0,0 +1,26 @@
+import { type Document, type ScrapeOptions } from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { ensureValidScrapeOptions } from "../utils/validation";
+import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";
+
+export async function scrape(http: HttpClient, url: string, options?: ScrapeOptions): Promise<Document> {
+ if (!url || !url.trim()) {
+ throw new Error("URL cannot be empty");
+ }
+ if (options) ensureValidScrapeOptions(options);
+
+ const payload: Record<string, unknown> = { url: url.trim() };
+ if (options) Object.assign(payload, options);
+
+ try {
+ const res = await http.post<{ success: boolean; data?: Document; error?: string }>("/v2/scrape", payload);
+ if (res.status !== 200 || !res.data?.success) {
+ throwForBadResponse(res, "scrape");
+ }
+ return (res.data.data || {}) as Document;
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");
+ throw err;
+ }
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
new file mode 100644
index 000000000..65e793140
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
@@ -0,0 +1,60 @@
+import { type Document, type SearchData, type SearchRequest, type SearchResult, type ScrapeOptions } from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { ensureValidScrapeOptions } from "../utils/validation";
+import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";
+
+function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
+ if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
+ if (req.limit != null && req.limit <= 0) throw new Error("limit must be positive");
+ if (req.timeout != null && req.timeout <= 0) throw new Error("timeout must be positive");
+ const payload: Record<string, unknown> = {
+ query: req.query,
+ };
+ if (req.sources) payload.sources = req.sources;
+ if (req.limit != null) payload.limit = req.limit;
+ if (req.tbs != null) payload.tbs = req.tbs;
+ if (req.location != null) payload.location = req.location;
+ if (req.ignoreInvalidURLs != null) payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
+ if (req.timeout != null) payload.timeout = req.timeout;
+ if (req.scrapeOptions) {
+ ensureValidScrapeOptions(req.scrapeOptions as ScrapeOptions);
+ payload.scrapeOptions = req.scrapeOptions;
+ }
+ return payload;
+}
+
+export async function search(http: HttpClient, request: SearchRequest): Promise<SearchData> {
+ const payload = prepareSearchPayload(request);
+ try {
+ const res = await http.post<{ success: boolean; data?: Record<string, unknown>; error?: string }>("/v2/search", payload);
+ if (res.status !== 200 || !res.data?.success) {
+ throwForBadResponse(res, "search");
+ }
+ const data = (res.data.data || {}) as Record<string, any>;
+ const out: SearchData = {};
+ for (const key of Object.keys(data)) {
+ const arr = data[key];
+ if (Array.isArray(arr)) {
+ const results: Array<SearchResult | Document> = [] as any;
+ for (const item of arr) {
+ if (item && typeof item === "object") {
+ // If scraped page fields present, treat as Document; otherwise SearchResult
+ if ("markdown" in item || "html" in item || "rawHtml" in item || "links" in item || "screenshot" in item || "changeTracking" in item) {
+ results.push(item as Document);
+ } else {
+ results.push({ url: item.url, title: item.title, description: item.description } as SearchResult);
+ }
+ } else if (typeof item === "string") {
+ results.push({ url: item } as SearchResult);
+ }
+ }
+ (out as any)[key] = results;
+ }
+ }
+ return out;
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "search");
+ throw err;
+ }
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/usage.ts b/apps/js-sdk/firecrawl/src/v2/methods/usage.ts
new file mode 100644
index 000000000..4138a70ec
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/methods/usage.ts
@@ -0,0 +1,39 @@
+import type { ConcurrencyCheck, CreditUsage, TokenUsage } from "../types";
+import { HttpClient } from "../utils/httpClient";
+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
+
+export async function getConcurrency(http: HttpClient): Promise<ConcurrencyCheck> {
+ try {
+ const res = await http.get<{ success: boolean; data?: { concurrency: number; maxConcurrency: number } }>("/v2/concurrency-check");
+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get concurrency");
+ const d = res.data.data || (res.data as any);
+ return { concurrency: d.concurrency, maxConcurrency: d.maxConcurrency ?? d.max_concurrency };
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "get concurrency");
+ throw err;
+ }
+}
+
+export async function getCreditUsage(http: HttpClient): Promise<CreditUsage> {
+ try {
+ const res = await http.get<{ success: boolean; data?: { remaining_credits: number } }>("/v2/team/credit-usage");
+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get credit usage");
+ const d = res.data.data || (res.data as any);
+ return { remaining_credits: d.remaining_credits };
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "get credit usage");
+ throw err;
+ }
+}
+
+export async function getTokenUsage(http: HttpClient): Promise<TokenUsage> {
+ try {
+ const res = await http.get<{ success: boolean; data?: TokenUsage }>("/v2/team/token-usage");
+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get token usage");
+ return (res.data.data || (res.data as any)) as TokenUsage;
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "get token usage");
+ throw err;
+ }
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
new file mode 100644
index 000000000..5dccff574
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -0,0 +1,305 @@
+import type { ZodTypeAny } from "zod";
+// Public types for Firecrawl JS/TS SDK v2 (camelCase only)
+
+export type FormatString =
+ | "markdown"
+ | "html"
+ | "rawHtml"
+ | "links"
+ | "screenshot"
+ | "summary"
+ | "changeTracking"
+ | "json";
+
+export interface Viewport {
+ width: number;
+ height: number;
+}
+
+export interface Format {
+ type: FormatString;
+}
+
+export interface JsonFormat extends Format {
+ type: "json";
+ prompt: string;
+ schema: Record<string, unknown> | ZodTypeAny;
+}
+
+export interface ScreenshotFormat {
+ type: "screenshot";
+ fullPage?: boolean;
+ quality?: number;
+ viewport?: Viewport | { width: number; height: number };
+}
+
+export interface ChangeTrackingFormat extends Format {
+ type: "changeTracking";
+ modes: ("git-diff" | "json")[];
+ schema?: Record<string, unknown>;
+ prompt?: string;
+ tag?: string;
+}
+
+export type FormatOption =
+ | FormatString
+ | Format
+ | JsonFormat
+ | ChangeTrackingFormat
+ | ScreenshotFormat;
+
+export interface LocationConfig {
+ country?: string;
+ languages?: string[];
+}
+
+export interface WaitAction {
+ type: "wait";
+ milliseconds?: number;
+ selector?: string;
+}
+
+export interface ScreenshotAction {
+ type: "screenshot";
+ fullPage?: boolean;
+ quality?: number;
+ viewport?: Viewport | { width: number; height: number };
+}
+
+export interface ClickAction {
+ type: "click";
+ selector: string;
+}
+
+export interface WriteAction {
+ type: "write";
+ text: string;
+}
+
+export interface PressAction {
+ type: "press";
+ key: string;
+}
+
+export interface ScrollAction {
+ type: "scroll";
+ direction: "up" | "down";
+ selector?: string;
+}
+
+export interface ScrapeAction {
+ type: "scrape";
+}
+
+export interface ExecuteJavascriptAction {
+ type: "executeJavascript";
+ script: string;
+}
+
+export interface PDFAction {
+ type: "pdf";
+ format?: "A0" | "A1" | "A2" | "A3" | "A4" | "A5" | "A6" | "Letter" | "Legal" | "Tabloid" | "Ledger";
+ landscape?: boolean;
+ scale?: number;
+}
+
+export type ActionOption =
+ | WaitAction
+ | ScreenshotAction
+ | ClickAction
+ | WriteAction
+ | PressAction
+ | ScrollAction
+ | ScrapeAction
+ | ExecuteJavascriptAction
+ | PDFAction;
+
+export interface ScrapeOptions {
+ formats?: FormatOption[];
+ headers?: Record<string, string>;
+ includeTags?: string[];
+ excludeTags?: string[];
+ onlyMainContent?: boolean;
+ timeout?: number;
+ waitFor?: number;
+ mobile?: boolean;
+ parsers?: string[];
+ actions?: ActionOption[];
+ location?: LocationConfig;
+ skipTlsVerification?: boolean;
+ removeBase64Images?: boolean;
+ fastMode?: boolean;
+ useMock?: string;
+ blockAds?: boolean;
+ proxy?: "basic" | "stealth" | "auto" | string;
+ maxAge?: number;
+ storeInCache?: boolean;
+}
+
+export interface WebhookConfig {
+ url: string;
+ headers?: Record<string, string>;
+ metadata?: Record<string, string>;
+ events?: Array<"completed" | "failed" | "page" | "started">;
+}
+
+export interface DocumentMetadata {
+ title?: string;
+ description?: string;
+ language?: string;
+ keywords?: string | string[];
+ robots?: string;
+ ogTitle?: string;
+ ogDescription?: string;
+ ogUrl?: string;
+ ogImage?: string;
+ sourceURL?: string;
+ statusCode?: number;
+ error?: string;
+ [key: string]: unknown;
+}
+
+export interface Document {
+ markdown?: string;
+ html?: string;
+ rawHtml?: string;
+ json?: unknown;
+ metadata?: DocumentMetadata;
+ links?: string[];
+ screenshot?: string;
+ actions?: Record<string, unknown>;
+ warning?: string;
+ changeTracking?: Record<string, unknown>;
+}
+
+export interface SearchResult {
+ url: string;
+ title?: string;
+ description?: string;
+}
+
+export interface SearchData {
+ web?: Array<SearchResult>;
+ news?: Array<SearchResult>;
+ images?: Array<SearchResult>;
+}
+
+export interface SearchRequest {
+ query: string;
+ sources?: Array<"web" | "news" | "images" | { type: "web" | "news" | "images" }>;
+ limit?: number;
+ tbs?: string;
+ location?: string;
+ ignoreInvalidURLs?: boolean;
+ timeout?: number; // ms
+ scrapeOptions?: ScrapeOptions;
+}
+
+export interface CrawlResponse {
+ id: string;
+ url: string;
+}
+
+export interface CrawlJob {
+ status: "scraping" | "completed" | "failed" | "cancelled";
+ total: number;
+ completed: number;
+ creditsUsed?: number;
+ expiresAt?: string;
+ next?: string | null;
+ data: Document[];
+}
+
+export interface BatchScrapeResponse {
+ id: string;
+ url: string;
+ invalidURLs?: string[];
+}
+
+export interface BatchScrapeJob {
+ status: "scraping" | "completed" | "failed" | "cancelled";
+ completed: number;
+ total: number;
+ creditsUsed?: number;
+ expiresAt?: string;
+ next?: string | null;
+ data: Document[];
+}
+
+export interface MapData {
+ links: SearchResult[];
+}
+
+export interface MapOptions {
+ search?: string;
+ sitemap?: "only" | "include" | "skip";
+ includeSubdomains?: boolean;
+ limit?: number;
+ timeout?: number;
+}
+
+export interface ExtractResponse {
+ success?: boolean;
+ id?: string;
+ status?: "processing" | "completed" | "failed" | "cancelled";
+ data?: unknown;
+ error?: string;
+ warning?: string;
+ sources?: Record<string, unknown>;
+ expiresAt?: string;
+}
+
+export interface ConcurrencyCheck {
+ concurrency: number;
+ maxConcurrency: number;
+}
+
+export interface CreditUsage {
+ remaining_credits: number;
+}
+
+export interface TokenUsage {
+ promptTokens: number;
+ completionTokens: number;
+ totalTokens: number;
+ step?: string;
+ model?: string;
+}
+
+export interface CrawlErrorsResponse {
+ errors: Array<Record<string, string>>;
+ robotsBlocked: string[];
+}
+
+export interface ActiveCrawl {
+ id: string;
+ teamId: string;
+ url: string;
+ options?: Record<string, unknown> | null;
+}
+
+export interface ActiveCrawlsResponse {
+ success: boolean;
+ crawls: ActiveCrawl[];
+}
+
+export interface ErrorDetails {
+ code?: string;
+ message: string;
+ details?: Record<string, unknown>;
+ status?: number;
+}
+
+export class SdkError extends Error {
+ status?: number;
+ code?: string;
+ details?: unknown;
+ constructor(message: string, status?: number, code?: string, details?: unknown) {
+ super(message);
+ this.name = "FirecrawlSdkError";
+ this.status = status;
+ this.code = code;
+ this.details = details;
+ }
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts
new file mode 100644
index 000000000..10adfc974
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts
@@ -0,0 +1,18 @@
+import { type AxiosError, type AxiosResponse } from "axios";
+import { SdkError } from "../types";
+
+export function throwForBadResponse(resp: AxiosResponse, action: string): never {
+ const status = resp.status;
+ const body = resp.data || {};
+ const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
+ throw new SdkError(msg, status, undefined, body?.details);
+}
+
+export function normalizeAxiosError(err: AxiosError, action: string): never {
+ const status = err.response?.status;
+ const body: any = err.response?.data;
+ const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
+ const code = (body?.code as string) || err.code;
+ throw new SdkError(message, status, code, body?.details ?? body);
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts
new file mode 100644
index 000000000..b646e9316
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts
@@ -0,0 +1,10 @@
+export function getVersion(): string {
+ try {
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
+ const pkg = require("../../package.json");
+ return (pkg?.version as string) || "3.0.0";
+ } catch {
+ return "3.0.0";
+ }
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
new file mode 100644
index 000000000..b2e57b487
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
@@ -0,0 +1,103 @@
+import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse } from "axios";
+import { getVersion } from "./getVersion";
+
+export interface HttpClientOptions {
+ apiKey: string;
+ apiUrl: string;
+ timeoutMs?: number;
+ maxRetries?: number;
+ backoffFactor?: number; // seconds factor for 0.5, 1, 2...
+}
+
+export class HttpClient {
+ private instance: AxiosInstance;
+ private readonly apiKey: string;
+ private readonly apiUrl: string;
+ private readonly maxRetries: number;
+ private readonly backoffFactor: number;
+
+ constructor(options: HttpClientOptions) {
+ this.apiKey = options.apiKey;
+ this.apiUrl = options.apiUrl.replace(/\/$/, "");
+ this.maxRetries = options.maxRetries ?? 3;
+ this.backoffFactor = options.backoffFactor ?? 0.5;
+ this.instance = axios.create({
+ baseURL: this.apiUrl,
+ timeout: options.timeoutMs ?? 60000,
+ headers: {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${this.apiKey}`,
+ },
+ transitional: { clarifyTimeoutError: true },
+ });
+ }
+
+ getApiUrl(): string {
+ return this.apiUrl;
+ }
+
+ getApiKey(): string {
+ return this.apiKey;
+ }
+
+ private async request<T = any>(config: AxiosRequestConfig): Promise<AxiosResponse<T>> {
+ const version = getVersion();
+ config.headers = {
+ ...(config.headers || {}),
+ // origin header for GET/DELETE; for POST we also include in body below
+ origin: `js-sdk@${version}`,
+ };
+
+ console.log("request", config);
+
+ let lastError: any;
+ for (let attempt = 0; attempt < this.maxRetries; attempt++) {
+ try {
+ const cfg: AxiosRequestConfig = { ...config };
+ // For POST/PUT, ensure origin is present in JSON body too
+ if (cfg.method && ["post", "put", "patch"].includes(cfg.method.toLowerCase())) {
+ const data = (cfg.data ?? {}) as Record<string, unknown>;
+ cfg.data = { ...data, origin: `js-sdk@${version}` };
+ }
+ const res = await this.instance.request<T>(cfg);
+ if (res.status === 502 && attempt < this.maxRetries - 1) {
+ await this.sleep(this.backoffFactor * Math.pow(2, attempt));
+ continue;
+ }
+ return res;
+ } catch (err: any) {
+ lastError = err;
+ const status = err?.response?.status;
+ if (status === 502 && attempt < this.maxRetries - 1) {
+ await this.sleep(this.backoffFactor * Math.pow(2, attempt));
+ continue;
+ }
+ throw err;
+ }
+ }
+ throw lastError ?? new Error("Unexpected HTTP client error");
+ }
+
+ private sleep(seconds: number): Promise<void> {
+ return new Promise((r) => setTimeout(r, seconds * 1000));
+ }
+
+ post<T = any>(endpoint: string, body: Record<string, unknown>, headers?: Record<string, string>) {
+ return this.request<T>({ method: "post", url: endpoint, data: body, headers });
+ }
+
+ get<T = any>(endpoint: string, headers?: Record<string, string>) {
+ return this.request<T>({ method: "get", url: endpoint, headers });
+ }
+
+ delete<T = any>(endpoint: string, headers?: Record<string, string>) {
+ return this.request<T>({ method: "delete", url: endpoint, headers });
+ }
+
+ prepareHeaders(idempotencyKey?: string): Record<string, string> {
+ const headers: Record<string, string> = {};
+ if (idempotencyKey) headers["x-idempotency-key"] = idempotencyKey;
+ return headers;
+ }
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
new file mode 100644
index 000000000..82c97ee35
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
@@ -0,0 +1,50 @@
+import { type FormatOption, type JsonFormat, type ScrapeOptions, type ScreenshotFormat } from "../types";
+import zodToJsonSchema from "zod-to-json-schema";
+
+export function ensureValidFormats(formats?: FormatOption[]): void {
+ if (!formats) return;
+ for (const fmt of formats) {
+ if (typeof fmt === "string") {
+ if (fmt === "json") {
+ throw new Error("json format must be an object with { type: 'json', prompt, schema }");
+ }
+ continue;
+ }
+ if ((fmt as JsonFormat).type === "json") {
+ const j = fmt as JsonFormat;
+ if (!j.prompt || !j.schema) {
+ throw new Error("json format requires 'prompt' and 'schema'");
+ }
+ // Flexibility: allow passing a Zod schema. Convert to JSON schema internally.
+ const maybeSchema: any = j.schema as any;
+ const isZod = !!maybeSchema && (typeof maybeSchema.safeParse === "function" || typeof maybeSchema.parse === "function") && !!maybeSchema._def;
+ if (isZod) {
+ try {
+ (j as any).schema = zodToJsonSchema(maybeSchema);
+ } catch {
+ // If conversion fails, leave as-is; server-side may still handle, or request will fail explicitly
+ }
+ }
+ continue;
+ }
+ if ((fmt as ScreenshotFormat).type === "screenshot") {
+ // no-op; already camelCase; validate numeric fields if present
+ const s = fmt as ScreenshotFormat;
+ if (s.quality != null && (typeof s.quality !== "number" || s.quality < 0)) {
+ throw new Error("screenshot.quality must be a non-negative number");
+ }
+ }
+ }
+}
+
+export function ensureValidScrapeOptions(options?: ScrapeOptions): void {
+ if (!options) return;
+ if (options.timeout != null && options.timeout <= 0) {
+ throw new Error("timeout must be positive");
+ }
+ if (options.waitFor != null && options.waitFor < 0) {
+ throw new Error("waitFor must be non-negative");
+ }
+ ensureValidFormats(options.formats);
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/watcher.ts b/apps/js-sdk/firecrawl/src/v2/watcher.ts
new file mode 100644
index 000000000..2e595c20d
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/watcher.ts
@@ -0,0 +1,153 @@
+import { EventEmitter } from "events";
+import type { BatchScrapeJob, CrawlJob, Document } from "./types";
+import type { HttpClient } from "./utils/httpClient";
+import { getBatchScrapeStatus } from "./methods/batch";
+import { getCrawlStatus } from "./methods/crawl";
+
+type JobKind = "crawl" | "batch";
+
+export interface WatcherOptions {
+ kind?: JobKind;
+ pollInterval?: number; // seconds
+ timeout?: number; // seconds
+}
+
+type Snapshot = CrawlJob | BatchScrapeJob;
+
+export class Watcher extends EventEmitter {
+ private readonly http: HttpClient;
+ private readonly jobId: string;
+ private readonly kind: JobKind;
+ private readonly pollInterval: number;
+ private readonly timeout?: number;
+ private ws?: WebSocket;
+ private closed = false;
+
+ constructor(http: HttpClient, jobId: string, opts: WatcherOptions = {}) {
+ super();
+ this.http = http;
+ this.jobId = jobId;
+ this.kind = opts.kind ?? "crawl";
+ this.pollInterval = opts.pollInterval ?? 2;
+ this.timeout = opts.timeout;
+ }
+
+ private buildWsUrl(): string {
+ // replace http/https with ws/wss
+ const apiUrl = this.http.getApiUrl();
+ const wsBase = apiUrl.replace(/^http/, "ws");
+ const path = this.kind === "crawl" ? `/v2/crawl/${this.jobId}` : `/v2/batch/scrape/${this.jobId}`;
+ return `${wsBase}${path}`;
+ }
+
+ async start(): Promise<void> {
+ try {
+ const url = this.buildWsUrl();
+ // Pass API key as subprotocol for browser compatibility
+ this.ws = new WebSocket(url, this.http.getApiKey());
+ this.attachWsHandlers(this.ws);
+ } catch {
+ // Fallback to polling immediately
+ this.pollLoop();
+ }
+ }
+
+ private attachWsHandlers(ws: WebSocket) {
+ let startTs = Date.now();
+ const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;
+ ws.onmessage = (ev: MessageEvent) => {
+ try {
+ const body = typeof ev.data === "string" ? JSON.parse(ev.data) : null;
+ if (!body) return;
+ const type = body.type as string | undefined;
+ if (type === "error") {
+ this.emit("error", { status: "failed", data: [], error: body.error, id: this.jobId });
+ return;
+ }
+ if (type === "catchup") {
+ const payload = body.data || {};
+ this.emitDocuments(payload.data || []);
+ this.emitSnapshot(payload);
+ return;
+ }
+ if (type === "document") {
+ const doc = body.data;
+ if (doc) this.emit("document", doc as Document & { id: string });
+ return;
+ }
+ if (type === "done") {
+ this.emit("done", { status: "completed", data: [], id: this.jobId });
+ this.close();
+ return;
+ }
+ const payload = body.data || body;
+ if (payload && payload.status) this.emitSnapshot(payload);
+ } catch {
+ // ignore
+ }
+ if (timeoutMs && Date.now() - startTs > timeoutMs) this.close();
+ };
+ ws.onerror = () => {
+ this.emit("error", { status: "failed", data: [], error: "WebSocket error", id: this.jobId });
+ this.close();
+ };
+ ws.onclose = () => {
+ if (!this.closed) this.pollLoop();
+ };
+ }
+
+ private emitDocuments(docs: Document[]) {
+ for (const doc of docs) this.emit("document", { ...(doc as any), id: this.jobId });
+ }
+
+ private emitSnapshot(payload: any) {
+ const status = payload.status as Snapshot["status"];
+ const data = (payload.data || []) as Document[];
+ const snap: Snapshot = this.kind === "crawl"
+ ? {
+ status,
+ completed: payload.completed ?? 0,
+ total: payload.total ?? 0,
+ creditsUsed: payload.creditsUsed,
+ expiresAt: payload.expiresAt,
+ next: payload.next ?? null,
+ data,
+ }
+ : {
+ status,
+ completed: payload.completed ?? 0,
+ total: payload.total ?? 0,
+ creditsUsed: payload.creditsUsed,
+ expiresAt: payload.expiresAt,
+ next: payload.next ?? null,
+ data,
+ };
+ this.emit("snapshot", snap);
+ if (["completed", "failed", "cancelled"].includes(status)) this.emit("done", { status, data, id: this.jobId });
+ }
+
+ private async pollLoop() {
+ const startTs = Date.now();
+ const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;
+ while (!this.closed) {
+ try {
+ const snap = this.kind === "crawl" ? await getCrawlStatus(this as any as HttpClient, this.jobId) : await getBatchScrapeStatus(this as any as HttpClient, this.jobId);
+ this.emit("snapshot", snap);
+ if (["completed", "failed", "cancelled"].includes(snap.status)) {
+ this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });
+ break;
+ }
+ } catch {
+ // ignore polling errors
+ }
+ if (timeoutMs && Date.now() - startTs > timeoutMs) break;
+ await new Promise((r) => setTimeout(r, Math.max(1000, this.pollInterval * 1000)));
+ }
+ }
+
+ close() {
+ this.closed = true;
+ if (this.ws && (this.ws as any).close) (this.ws as any).close();
+ }
+}
+
diff --git a/apps/js-sdk/firecrawl/tsconfig.json b/apps/js-sdk/firecrawl/tsconfig.json
index 1297aed94..dbfc40ca2 100644
--- a/apps/js-sdk/firecrawl/tsconfig.json
+++ b/apps/js-sdk/firecrawl/tsconfig.json
@@ -17,7 +17,8 @@
"noImplicitOverride": true,
/* If NOT transpiling with TypeScript: */
- "module": "NodeNext",
+ "module": "ESNext",
+ "moduleResolution": "Bundler",
"noEmit": true,
},
"include": ["src/**/*"],
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py
index 68a08aa3e..a1468d44e 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py
@@ -38,11 +38,12 @@ def test_search_minimal_request():
assert isinstance(results, SearchData)
assert hasattr(results, 'web')
- assert hasattr(results, 'news')
- assert hasattr(results, 'images')
-
assert results.web is not None
assert len(results.web) > 0
+ assert hasattr(results, 'news')
+ assert results.news is None
+ assert hasattr(results, 'images')
+ assert results.images is None
for result in results.web:
assert isinstance(result, SearchResult)