Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(js-sdk): methods done. todo: e2e/unit tests
rafaelmmillerec69c309920b
diff --git a/apps/js-sdk/.env.example b/apps/js-sdk/.env.examplenew file mode 100644index 000000000..81394d178--- /dev/null+++ b/apps/js-sdk/.env.example@@ -0,0 +1,2 @@+FIRECRAWL_API_KEY=<your api key>+FIRECRAWL_API_URL=https://api.firecrawl.dev\ No newline at end of filediff --git a/apps/js-sdk/example.js b/apps/js-sdk/example.jsindex 21bdb2a1b..d473743f4 100644--- a/apps/js-sdk/example.js+++ b/apps/js-sdk/example.js@@ -1,3 +1,38 @@+// Placeholder v2 example (JavaScript)+// Minimal usage of new FirecrawlClient. Replace with your API key before running.++import { FirecrawlClient } from './firecrawl/src/v2/client.js';++async function run() {+ const apiKey = (globalThis.process && globalThis.process.env && globalThis.process.env.FIRECRAWL_API_KEY) || 'fc-YOUR_API_KEY';+ const client = new FirecrawlClient({ apiKey });++ const doc = await client.scrape('https://docs.firecrawl.dev', { formats: ['markdown'] });+ console.log('scrape:', !!doc.markdown);++ const crawl = await client.crawl('https://docs.firecrawl.dev', { limit: 3, pollInterval: 1, timeout: 120 });+ console.log('crawl:', crawl.status, crawl.completed, '/', crawl.total);++ const batch = await client.batchScrape([+ 'https://docs.firecrawl.dev',+ 'https://firecrawl.dev',+ ], { options: { formats: ['markdown'] }, pollInterval: 1, timeout: 120 });+ console.log('batch:', batch.status, batch.completed, '/', batch.total);++ const search = await client.search('What is the capital of France?', { limit: 5 });+ console.log('search web results:', (search.web || []).length);++ const map = await client.map('https://firecrawl.dev');+ console.log('map links:', map.links.length);+}++run().catch((e) => {+ console.error(e);+ if (globalThis.process && globalThis.process.exit) globalThis.process.exit(1);+});++// old stuff:+import FirecrawlApp from 'firecrawl';import { z } from 'zod';diff --git a/apps/js-sdk/example.ts b/apps/js-sdk/example.tsindex a8fff30a2..815c0c086 100644--- a/apps/js-sdk/example.ts+++ b/apps/js-sdk/example.ts@@ -1,3 +1,41 @@+// Placeholder v2 example (TypeScript)+// Minimal usage of new FirecrawlClient. Replace with your API key before running.++// import Firecrawl from '@mendable/firecrawl-js';+import Firecrawl from './firecrawl/src/index';++const run = async () => {+ const apiKey = (globalThis as any).process?.env?.FIRECRAWL_API_KEY || 'fc-YOUR_API_KEY';+ const apiUrl = (globalThis as any).process?.env?.FIRECRAWL_API_URL || 'https://api.firecrawl.dev';+ const client = new Firecrawl({ apiKey, apiUrl });++ const doc = await client.scrape('https://docs.firecrawl.dev', { formats: ['markdown'] });+ console.log('scrape:', !!doc.markdown);++ const crawl = await client.crawl('https://docs.firecrawl.dev', { limit: 3, pollInterval: 1, timeout: 120 });+ console.log('crawl:', crawl.status, crawl.completed, '/', crawl.total);++ const batch = await client.batchScrape([+ 'https://docs.firecrawl.dev',+ 'https://firecrawl.dev',+ ], { options: { formats: ['markdown'] }, pollInterval: 1, timeout: 120 });+ console.log('batch:', batch.status, batch.completed, '/', batch.total);++ const search = await client.search('What is the capital of France?', { limit: 5 });+ console.log('search web results:', (search.web || []).length);++ const map = await client.map('https://firecrawl.dev');+ console.log('map links:', map.links.length);+};++run().catch((e) => {+ // eslint-disable-next-line no-console+ console.error(e);+ (globalThis as any).process?.exit?.(1);+});++// old stuff:+import FirecrawlApp, { CrawlStatusResponse, ErrorResponse } from 'firecrawl';const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});diff --git a/apps/js-sdk/example_v1.js b/apps/js-sdk/example_v1.jsnew file mode 100644index 000000000..7700e68af--- /dev/null+++ b/apps/js-sdk/example_v1.js@@ -0,0 +1,19 @@+import FirecrawlApp from 'firecrawl';++// Placeholder v1 example (JavaScript)+// Mirrors the older SDK usage. Replace with your API key before running.++async function main() {+ const app = new Firecrawl({ apiKey: process.env.FIRECRAWL_API_KEY || 'fc-YOUR_API_KEY' });++ const scrape = await app.v1.scrapeUrl('firecrawl.dev');+ if (scrape && scrape.success) console.log(scrape.markdown);++ const crawl = await app.v1.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 3 });+ console.log(crawl);+}++main().catch((e) => {+ console.error(e);+ process.exit(1);+});\ No newline at end of filediff --git a/apps/js-sdk/example_v1.ts b/apps/js-sdk/example_v1.tsnew file mode 100644index 000000000..d845a73ef--- /dev/null+++ b/apps/js-sdk/example_v1.ts@@ -0,0 +1,23 @@+// Placeholder v1 example (TypeScript)+// Mirrors the older SDK usage. Replace with your API key before running.++// import FirecrawlApp from 'firecrawl';+import Firecrawl from './firecrawl/src/index'++async function main() {+ const app = new Firecrawl({ apiKey: process.env.FIRECRAWL_API_KEY || 'fc-YOUR_API_KEY' });++ // Scrape a website (v1 style):+ const scrape = await app.v1.scrapeUrl('firecrawl.dev');+ if ((scrape as any).success) console.log((scrape as any).markdown);++ // Crawl a website (v1 style):+ const crawl = await app.v1.crawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 3 });+ console.log(crawl);+}++main().catch((e) => {+ console.error(e);+ process.exit(1);+});+diff --git a/apps/js-sdk/example_watcher.ts b/apps/js-sdk/example_watcher.tsnew file mode 100644index 000000000..b163b82b5--- /dev/null+++ b/apps/js-sdk/example_watcher.ts@@ -0,0 +1,40 @@+/*+ Advanced watcher example using the v2 FirecrawlClient.++ Run with:+ node --env-file=.env -r esbuild-register apps/js-sdk/firecrawl/src/examples/watcher.ts+ or compile with your bundler, ensuring FIRECRAWL_API_KEY is set.+*/++import { FirecrawlClient } from "./firecrawl/src/v2/client";++async function main() {+ const apiKey = process.env.FIRECRAWL_API_KEY || "fc-YOUR_API_KEY";+ const client = new FirecrawlClient({ apiKey });++ // Start a crawl and attach a watcher for real-time updates+ const start = await client.startCrawl("https://example.com", { limit: 5 });+ console.log("Started crawl:", start.id);++ const watcher = client.watcher(start.id, { kind: "crawl", pollInterval: 2, timeout: 120 });++ watcher.on("document", (doc) => {+ console.log("document:", (doc as any).url || (doc as any).metadata?.sourceURL || "<no url>");+ });++ watcher.on("snapshot", (snap) => {+ console.log(`status: ${snap.status} (${snap.completed}/${snap.total})`);+ });++ watcher.on("done", (finalState) => {+ console.log("done:", finalState.status, "docs:", finalState.data?.length ?? 0);+ });++ await watcher.start();+}++main().catch((e) => {+ console.error(e);+ process.exit(1);+});+diff --git a/apps/js-sdk/firecrawl/.env.example b/apps/js-sdk/firecrawl/.env.exampleindex 6b1780bb8..81394d178 100644--- a/apps/js-sdk/firecrawl/.env.example+++ b/apps/js-sdk/firecrawl/.env.example@@ -1,3 +1,2 @@-API_URL=http://localhost:3002-TEST_API_KEY=fc-YOUR_API_KEY-+FIRECRAWL_API_KEY=<your api key>+FIRECRAWL_API_URL=https://api.firecrawl.dev\ No newline at end of filediff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.jsonindex 76a7d087d..02897e3fa 100644--- a/apps/js-sdk/firecrawl/package-lock.json+++ b/apps/js-sdk/firecrawl/package-lock.json@@ -1,12 +1,12 @@{"name": "@mendable/firecrawl-js",- "version": "1.29.1",+ "version": "1.29.3","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@mendable/firecrawl-js",- "version": "1.29.1",+ "version": "1.29.3","license": "MIT","dependencies": {"axios": "^1.11.0",diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 1aa0a0420..3b38b1fd8 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "1.29.3",+ "version": "3.0.1","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",@@ -16,8 +16,8 @@"build": "tsup","build-and-publish": "npm run build && npm publish --access public","publish-beta": "npm run build && npm publish --access public --tag beta",- "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts",- "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/unit/*.test.ts"+ "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/e2e/v2/**/*.test.ts",+ "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/unit/v2/*.test.ts"},"repository": {"type": "git",diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v1/index.test.tssimilarity index 100%rename from apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.tsrename to apps/js-sdk/firecrawl/src/__tests__/e2e/v1/index.test.tsdiff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.tsnew file mode 100644index 000000000..3b3752896--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/scrape.test.ts@@ -0,0 +1,122 @@+/**+ * E2E tests for v2 scrape+ */+import Firecrawl from "../../../../index";+import { z } from "zod";+import { config } from "dotenv";+import { describe, test, expect } from "@jest/globals";++config();++const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";++describe("v2.scrape e2e", () => {+ if (!API_KEY) {+ console.warn("Skipping v2.scrape e2e: FIRECRAWL_API_KEY not set");+ }++ const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL })++ const assertValidDocument = (doc: any) => {+ expect(doc).toBeTruthy();+ const hasContent = Boolean(doc.markdown?.length) || Boolean(doc.html?.length) || Boolean(doc.rawHtml?.length);+ expect(hasContent).toBe(true);+ expect(doc.metadata).toBeTruthy();+ };++ test("minimal: scrape only required params", async () => {+ if (!client) throw new Error();+ const doc = await client.scrape("https://docs.firecrawl.dev");+ assertValidDocument(doc);+ }, 60_000);++ test("maximal: scrape with all options", async () => {+ if (!client) throw new Error();+ const doc = await client.scrape("https://docs.firecrawl.dev", {+ formats: [+ "markdown",+ "html",+ "rawHtml",+ "links",+ { type: "screenshot", fullPage: true, quality: 80, viewport: { width: 1280, height: 800 } },+ {+ type: "json",+ prompt: "Summarize the page and list links",+ schema: {+ type: "object",+ properties: {+ summary: { type: "string" },+ links: { type: "array", items: { type: "string", format: "uri" } },+ },+ required: ["summary"],+ },+ },+ ],+ parsers: ["pdf"],+ headers: { "User-Agent": "firecrawl-tests" },+ includeTags: ["article"],+ excludeTags: ["nav"],+ onlyMainContent: true,+ waitFor: 1000,+ timeout: 30_000,+ location: { country: "us", languages: ["en"] },+ mobile: false,+ skipTlsVerification: false,+ removeBase64Images: true,+ blockAds: true,+ proxy: "auto",+ storeInCache: true,+ maxAge: 60_000,+ });+ assertValidDocument(doc);+ }, 90_000);++ test("json format with zod schema (auto-converted internally)", async () => {+ if (!client) throw new Error();+ const zodSchema = z.object({+ title: z.string().min(1),+ items: z.array(z.string().url()).optional(),+ });++ const doc = await client.scrape("https://docs.firecrawl.dev", {+ formats: [+ {+ type: "json",+ prompt: "Extract title and items",+ schema: zodSchema,+ },+ ],+ });+ expect(doc).toBeTruthy();+ }, 90_000);++ test.each([+ ["markdown", "markdown"],+ ["html", "html"],+ ["rawHtml", "rawHtml"],+ ["links", "links"],+ ["screenshot", "screenshot"],+ ])("basic format: %s", async (fmt, expectField) => {+ if (!client) throw new Error();+ const doc = await client.scrape("https://docs.firecrawl.dev", { formats: [fmt as any] });+ if (expectField !== "links" && expectField !== "screenshot") {+ assertValidDocument(doc);+ }+ if (expectField === "markdown") expect(doc.markdown).toBeTruthy();+ if (expectField === "html") expect(doc.html).toBeTruthy();+ if (expectField === "rawHtml") expect(doc.rawHtml).toBeTruthy();+ if (expectField === "screenshot") expect(doc.screenshot).toBeTruthy();+ if (expectField === "links") {+ expect(Array.isArray(doc.links)).toBe(true);+ expect((doc.links || []).length).toBeGreaterThan(0);+ }+ }, 90_000);++ test("invalid url should throw", async () => {+ if (!client) throw new Error();+ await expect(client.scrape("")).rejects.toThrow("URL cannot be empty");+ await expect(client.scrape(" ")).rejects.toThrow("URL cannot be empty");+ });+});+diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.tsnew file mode 100644index 000000000..80871eea5--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/methods/search.test.ts@@ -0,0 +1,210 @@+/**+ * E2E tests for v2 search (translated from Python tests)+ */+import Firecrawl from "../../../../index";+import { config } from "dotenv";+import { describe, test, expect } from "@jest/globals";++config();++const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";+const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";++const client = API_KEY ? new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL }) : null as any;++function collectTexts(entries: any[] | undefined): string[] {+ const texts: string[] = [];+ for (const r of entries || []) {+ const title = (r && typeof r === 'object') ? (r.title as unknown as string | undefined) : undefined;+ const desc = (r && typeof r === 'object') ? (r.description as unknown as string | undefined) : undefined;+ if (title) texts.push(String(title).toLowerCase());+ if (desc) texts.push(String(desc).toLowerCase());+ }+ return texts;+}++function isDocument(entry: any): boolean {+ return !!entry && (typeof entry.markdown === 'string' || typeof entry.html === 'string');+}++describe("v2.search e2e", () => {+ if (!client) {+ console.warn("Skipping v2.search e2e: FIRECRAWL_API_KEY not set");+ }++ test("minimal request", async () => {+ if (!client) throw new Error();+ const results = await client.search("What is the capital of France?");+ expect(results).toBeTruthy();+ expect(results).toHaveProperty("web");+ expect(results).not.toHaveProperty("news");+ expect(results).not.toHaveProperty("images");++ expect(results.web).toBeTruthy();+ expect((results.web || []).length).toBeGreaterThan(0);++ for (const result of results.web || []) {+ if (isDocument(result)) {+ // documents appear if scraping happens+ continue;+ }+ expect(typeof result.url).toBe("string");+ expect(result.url.startsWith("http")).toBe(true);+ expect(typeof result.title === "string" || result.title == null).toBe(true);+ expect(typeof result.description === "string" || result.description == null).toBe(true);+ }++ const allText = collectTexts(results.web).join(" ");+ expect(allText.includes("paris")).toBe(true);++ expect(results.news == null).toBe(true);+ expect(results.images == null).toBe(true);+ }, 90_000);++ test("with sources web+news and limit", async () => {+ if (!client) throw new Error();+ const results = await client.search("firecrawl", { sources: ["web", "news"], limit: 3 });+ expect(results).toBeTruthy();+ expect(results.web).toBeTruthy();+ expect((results.web || []).length).toBeLessThanOrEqual(3);+ if (results.news != null) {+ expect((results.news || []).length).toBeLessThanOrEqual(3);+ }+ expect(results.images == null).toBe(true);++ const webTitles = (results.web || []).map(r => (r?.title || "").toString().toLowerCase());+ const webDescriptions = (results.web || []).map(r => (r?.description || "").toString().toLowerCase());+ const allWebText = (webTitles.concat(webDescriptions)).join(" ");+ expect(allWebText.includes("firecrawl")).toBe(true);+ }, 90_000);++ test("result structure", async () => {+ if (!client) throw new Error();+ const results = await client.search("test query", { limit: 1 });+ if (results.web && results.web.length > 0) {+ const result: any = results.web[0];+ expect(result).toHaveProperty("url");+ expect(result).toHaveProperty("title");+ expect(result).toHaveProperty("description");+ expect(typeof result.url).toBe("string");+ expect(typeof result.title === "string" || result.title == null).toBe(true);+ expect(typeof result.description === "string" || result.description == null).toBe(true);+ expect(result.url.startsWith("http")).toBe(true);+ }+ }, 90_000);++ test("all parameters (comprehensive)", async () => {+ if (!client) throw new Error();+ const schema = {+ type: "object",+ properties: {+ title: { type: "string" },+ description: { type: "string" },+ url: { type: "string" },+ },+ required: ["title", "description"],+ } as const;++ const results = await client.search("artificial intelligence", {+ sources: [ "web", "news", "images" ],+ limit: 3,+ tbs: "qdr:m",+ location: "US",+ ignoreInvalidURLs: true,+ timeout: 60_000,+ scrapeOptions: {+ formats: [+ "markdown",+ "html",+ { type: "json", prompt: "Extract the title and description from the page", schema },+ ],+ headers: { "User-Agent": "Firecrawl-Test/1.0" },+ includeTags: ["h1", "h2", "p"],+ excludeTags: ["nav", "footer"],+ onlyMainContent: true,+ waitFor: 2000,+ mobile: false,+ skipTlsVerification: false,+ removeBase64Images: true,+ blockAds: true,+ proxy: "basic",+ maxAge: 3_600_000,+ storeInCache: true,+ location: { country: "US", languages: ["en"] },+ actions: [{ type: "wait", milliseconds: 1000 }],+ },+ });++ expect(results).toBeTruthy();+ expect(results).toHaveProperty("web");+ expect(results).toHaveProperty("news");+ expect(results).toHaveProperty("images");++ expect(results.web).toBeTruthy();+ expect((results.web || []).length).toBeLessThanOrEqual(3);++ const nonDocEntries = (results.web || []).filter(r => !isDocument(r));+ if (nonDocEntries.length > 0) {+ const allWebText = collectTexts(nonDocEntries).join(" ");+ const aiTerms = ["artificial", "intelligence", "ai", "machine", "learning"];+ expect(aiTerms.some(t => allWebText.includes(t))).toBe(true);+ }++ for (const result of results.web || []) {+ if (isDocument(result)) {+ expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true);+ } else {+ expect(typeof result.url).toBe("string");+ expect(result.url.startsWith("http")).toBe(true);+ }+ }++ if (results.news != null) {+ expect((results.news || []).length).toBeLessThanOrEqual(3);+ for (const result of results.news || []) {+ if (isDocument(result)) {+ expect(Boolean(result.markdown) || Boolean(result.html)).toBe(true);+ } else {+ expect(typeof result.url).toBe("string");+ expect(result.url.startsWith("http")).toBe(true);+ }+ }+ }++ expect(results.images == null).toBe(true);+ }, 120_000);++ test("formats flexibility: list vs object", async () => {+ if (!client) throw new Error();+ const results1 = await client.search("python programming", {+ limit: 1,+ scrapeOptions: { formats: ["markdown"] },+ });+ const results2 = await client.search("python programming", {+ limit: 1,+ scrapeOptions: { formats: [{ type: "markdown" as any }] },+ });+ expect(results1).toBeTruthy();+ expect(results2).toBeTruthy();+ expect(results1.web).toBeTruthy();+ expect(results2.web).toBeTruthy();+ }, 90_000);++ test("with json format object", async () => {+ if (!client) throw new Error();+ const jsonSchema = {+ type: "object",+ properties: { title: { type: "string" } },+ required: ["title"],+ } as const;+ const results = await client.search("site:docs.firecrawl.dev", {+ limit: 1,+ scrapeOptions: {+ formats: [{ type: "json", prompt: "Extract page title", schema: jsonSchema }],+ },+ });+ expect(results).toBeTruthy();+ expect(Array.isArray(results.web) || results.web == null).toBe(true);+ }, 90_000);+});+diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v1/monitor-job-status-retry.test.tssimilarity index 100%rename from apps/js-sdk/firecrawl/src/__tests__/v1/unit/monitor-job-status-retry.test.tsrename to apps/js-sdk/firecrawl/src/__tests__/unit/v1/monitor-job-status-retry.test.tsdiff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape.unit.test.tsnew file mode 100644index 000000000..1784cdc85--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/scrape.unit.test.ts@@ -0,0 +1,11 @@+/**+ * Minimal unit test for v2 scrape (no mocking; sanity check payload path)+ */+import { FirecrawlClient } from "../../../v2/client";++describe("v2.scrape unit", () => {+ test("constructor requires apiKey", () => {+ expect(() => new FirecrawlClient({ apiKey: "", apiUrl: "https://api.firecrawl.dev" })).toThrow();+ });+});+diff --git a/apps/js-sdk/firecrawl/src/index.backup.ts b/apps/js-sdk/firecrawl/src/index.backup.tsnew file mode 100644index 000000000..f93bb48e3--- /dev/null+++ b/apps/js-sdk/firecrawl/src/index.backup.ts@@ -0,0 +1,2146 @@+import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios";+import * as zt from "zod";+import { zodToJsonSchema } from "zod-to-json-schema";+import { TypedEventTarget } from "typescript-event-target";++/**+ * Configuration interface for FirecrawlApp.+ * @param apiKey - Optional API key for authentication.+ * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.+ */+export interface FirecrawlAppConfig {+ apiKey?: string | null;+ apiUrl?: string | null;+}++/**+ * Metadata for a Firecrawl document.+ * Includes various optional properties for document metadata.+ */+export interface FirecrawlDocumentMetadata {+ title?: string;+ description?: string;+ language?: string;+ keywords?: string;+ robots?: string;+ ogTitle?: string;+ ogDescription?: string;+ ogUrl?: string;+ ogImage?: string;+ ogAudio?: string;+ ogDeterminer?: string;+ ogLocale?: string;+ ogLocaleAlternate?: string[];+ ogSiteName?: string;+ ogVideo?: string;+ dctermsCreated?: string;+ dcDateCreated?: string;+ dcDate?: string;+ dctermsType?: string;+ dcType?: string;+ dctermsAudience?: string;+ dctermsSubject?: string;+ dcSubject?: string;+ dcDescription?: string;+ dctermsKeywords?: string;+ modifiedTime?: string;+ publishedTime?: string;+ articleTag?: string;+ articleSection?: string;+ sourceURL?: string;+ statusCode?: number;+ error?: string;+ proxyUsed?: "basic" | "stealth";+ cacheState?: "miss" | "hit";+ cachedAt?: string;+ [key: string]: any; // Allows for additional metadata properties not explicitly defined.+}++/**+ * Document interface for Firecrawl.+ * Represents a document retrieved or processed by Firecrawl.+ */+export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | never) = never> {+ url?: string;+ markdown?: string;+ html?: string;+ rawHtml?: string;+ links?: string[];+ extract?: T;+ json?: T;+ screenshot?: string;+ metadata?: FirecrawlDocumentMetadata;+ actions: ActionsSchema;+ changeTracking?: {+ previousScrapeAt: string | null;+ changeStatus: "new" | "same" | "changed" | "removed";+ visibility: "visible" | "hidden";+ diff?: {+ text: string;+ json: {+ files: Array<{+ from: string | null;+ to: string | null;+ chunks: Array<{+ content: string;+ changes: Array<{+ type: string;+ normal?: boolean;+ ln?: number;+ ln1?: number;+ ln2?: number;+ content: string;+ }>;+ }>;+ }>;+ };+ };+ json?: any;+ };+ // v1 search only+ title?: string;+ description?: string;+}++/**+ * Parameters for scraping operations.+ * Defines the options and configurations available for scraping web content.+ */+export interface CrawlScrapeOptions {+ formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "changeTracking")[];+ headers?: Record<string, string>;+ includeTags?: string[];+ excludeTags?: string[];+ onlyMainContent?: boolean;+ waitFor?: number;+ timeout?: number;+ location?: {+ country?: string;+ languages?: string[];+ };+ mobile?: boolean;+ skipTlsVerification?: boolean;+ removeBase64Images?: boolean;+ blockAds?: boolean;+ proxy?: "basic" | "stealth" | "auto";+ storeInCache?: boolean;+ maxAge?: number;+ parsePDF?: boolean;+}++export type Action = {+ type: "wait",+ milliseconds?: number,+ selector?: string,+} | {+ type: "click",+ selector: string,+ all?: boolean,+} | {+ type: "screenshot",+ fullPage?: boolean,+ quality?: number,+} | {+ type: "write",+ text: string,+} | {+ type: "press",+ key: string,+} | {+ type: "scroll",+ direction?: "up" | "down",+ selector?: string,+} | {+ type: "scrape",+} | {+ type: "executeJavascript",+ script: string,+};++export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema extends (Action[] | undefined) = undefined> extends CrawlScrapeOptions {+ extract?: {+ prompt?: string;+ schema?: LLMSchema;+ systemPrompt?: string;+ };+ jsonOptions?:{+ prompt?: string;+ schema?: LLMSchema;+ systemPrompt?: string;+ }+ changeTrackingOptions?: {+ prompt?: string;+ schema?: any;+ modes?: ("json" | "git-diff")[];+ tag?: string | null;+ }+ actions?: ActionsSchema;+ agent?: AgentOptions;+ zeroDataRetention?: boolean;+}++export interface ActionsResult {+ screenshots: string[];+ scrapes: ({+ url: string;+ html: string;+ })[];+ javascriptReturns: {+ type: string;+ value: unknown+ }[];+}++/**+ * Response interface for scraping operations.+ * Defines the structure of the response received after a scraping operation.+ */+export interface ScrapeResponse<LLMResult = any, ActionsSchema extends (ActionsResult | never) = never> extends FirecrawlDocument<LLMResult, ActionsSchema> {+ success: true;+ warning?: string;+ error?: string;+}++/**+ * Parameters for crawling operations.+ * Includes options for both scraping and mapping during a crawl.+ */+export interface CrawlParams {+ includePaths?: string[];+ excludePaths?: string[];+ maxDepth?: number;+ maxDiscoveryDepth?: number;+ limit?: number;+ allowBackwardLinks?: boolean;+ crawlEntireDomain?: boolean;+ allowExternalLinks?: boolean;+ ignoreSitemap?: boolean;+ scrapeOptions?: CrawlScrapeOptions;+ webhook?: string | {+ url: string;+ headers?: Record<string, string>;+ metadata?: Record<string, string>;+ events?: ["completed", "failed", "page", "started"][number][];+ };+ deduplicateSimilarURLs?: boolean;+ ignoreQueryParameters?: boolean;+ regexOnFullURL?: boolean;+ /**+ * Delay in seconds between scrapes. This helps respect website rate limits.+ * If not provided, the crawler may use the robots.txt crawl delay if available.+ */+ delay?: number;+ allowSubdomains?: boolean;+ maxConcurrency?: number;+ zeroDataRetention?: boolean;+}++/**+ * Response interface for crawling operations.+ * Defines the structure of the response received after initiating a crawl.+ */+export interface CrawlResponse {+ id?: string;+ url?: string;+ success: true;+ error?: string;+}++/**+ * Response interface for batch scrape operations.+ * Defines the structure of the response received after initiating a crawl.+ */+export interface BatchScrapeResponse {+ id?: string;+ url?: string;+ success: true;+ error?: string;+ invalidURLs?: string[];+}++/**+ * Response interface for job status checks.+ * Provides detailed status of a crawl job including progress and results.+ */+export interface CrawlStatusResponse {+ success: true;+ status: "scraping" | "completed" | "failed" | "cancelled";+ completed: number;+ total: number;+ creditsUsed: number;+ expiresAt: Date;+ next?: string;+ data: FirecrawlDocument<undefined>[];+};++/**+ * Response interface for batch scrape job status checks.+ * Provides detailed status of a batch scrape job including progress and results.+ */+export interface BatchScrapeStatusResponse {+ success: true;+ status: "scraping" | "completed" | "failed" | "cancelled";+ completed: number;+ total: number;+ creditsUsed: number;+ expiresAt: Date;+ next?: string;+ data: FirecrawlDocument<undefined>[];+};++/**+ * Parameters for mapping operations.+ * Defines options for mapping URLs during a crawl.+ */+export interface MapParams {+ search?: string;+ ignoreSitemap?: boolean;+ includeSubdomains?: boolean;+ sitemapOnly?: boolean;+ limit?: number;+ timeout?: number;+ useIndex?: boolean;+}++/**+ * Response interface for mapping operations.+ * Defines the structure of the response received after a mapping operation.+ */+export interface MapResponse {+ success: true;+ links?: string[];+ error?: string;+}++/**+ * Parameters for extracting information from URLs.+ * Defines options for extracting information from URLs.+ */+export interface AgentOptions {+ model?: string;+ prompt?: string;+ sessionId?: string;+}++/**+ * Parameters for extracting information from URLs.+ * Defines options for extracting information from URLs.+ */+export interface AgentOptionsExtract {+ model?: string;+ sessionId?: string;+}++export interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {+ prompt?: string;+ schema?: LLMSchema | object;+ systemPrompt?: string;+ allowExternalLinks?: boolean;+ enableWebSearch?: boolean;+ includeSubdomains?: boolean;+ origin?: string;+ showSources?: boolean;+ scrapeOptions?: CrawlScrapeOptions;+ agent?: AgentOptionsExtract;+}++/**+ * Response interface for extracting information from URLs.+ * Defines the structure of the response received after extracting information from URLs.+ */+export interface ExtractResponse<LLMSchema extends zt.ZodSchema = any> {+ success: boolean;+ data: LLMSchema;+ error?: string;+ warning?: string;+ sources?: string[];+}++/**+ * Error response interface.+ * Defines the structure of the response received when an error occurs.+ */+export interface ErrorResponse {+ success: false;+ error: string;+}++/**+ * Custom error class for Firecrawl.+ * Extends the built-in Error class to include a status code.+ */+export class FirecrawlError extends Error {+ statusCode: number;+ details?: any;+ constructor(message: string, statusCode: number, details?: any) {+ super(message);+ this.statusCode = statusCode;+ this.details = details;+ }+}++/**+ * Parameters for search operations.+ * Defines options for searching and scraping search results.+ */+export interface SearchParams {+ limit?: number;+ tbs?: string;+ filter?: string;+ lang?: string;+ country?: string;+ location?: string;+ origin?: string;+ timeout?: number;+ scrapeOptions?: ScrapeParams;+}++/**+ * Response interface for search operations.+ * Defines the structure of the response received after a search operation.+ */+export interface SearchResponse {+ success: boolean;+ data: FirecrawlDocument<undefined>[];+ warning?: string;+ error?: string;+}++/**+ * Response interface for crawl/batch scrape error monitoring.+ */+export interface CrawlErrorsResponse {+ /**+ * Scrapes that errored out + error details+ */+ errors: {+ id: string,+ timestamp?: string,+ url: string,+ error: string,+ }[];++ /**+ * URLs blocked by robots.txt+ */+ robotsBlocked: string[];+};++/**+ * Parameters for deep research operations.+ * Defines options for conducting deep research on a query.+ */+export interface DeepResearchParams<LLMSchema extends zt.ZodSchema = any> {+ /**+ * Maximum depth of research iterations (1-10)+ * @default 7+ */+ maxDepth?: number;+ /**+ * Time limit in seconds (30-300)+ * @default 270+ */+ timeLimit?: number;+ /**+ * Maximum number of URLs to analyze (1-1000)+ * @default 20+ */+ maxUrls?: number;+ /**+ * The prompt to use for the final analysis+ */+ analysisPrompt?: string;+ /**+ * The system prompt to use for the research agent+ */+ systemPrompt?: string;+ /**+ * The formats to use for the final analysis+ */+ formats?: ("markdown" | "json")[];+ /**+ * The JSON options to use for the final analysis+ */+ jsonOptions?:{+ prompt?: string;+ schema?: LLMSchema;+ systemPrompt?: string;+ };+ /**+ * Experimental flag for streaming steps+ */+ // __experimental_streamSteps?: boolean;+}++/**+ * Response interface for deep research operations.+ */+export interface DeepResearchResponse {+ success: boolean;+ id: string;+}++/**+ * Status response interface for deep research operations.+ */+export interface DeepResearchStatusResponse {+ success: boolean;+ data: {+ finalAnalysis: string;+ activities: Array<{+ type: string;+ status: string;+ message: string;+ timestamp: string;+ depth: number;+ }>;+ sources: Array<{+ url: string;+ title: string;+ description: string;+ }>;+ };+ status: "processing" | "completed" | "failed";+ error?: string;+ expiresAt: string;+ currentDepth: number;+ maxDepth: number;+ activities: Array<{+ type: string;+ status: string;+ message: string;+ timestamp: string;+ depth: number;+ }>;+ sources: Array<{+ url: string;+ title: string;+ description: string;+ }>;+ summaries: string[];+}++/**+ * Parameters for LLMs.txt generation operations.+ */+export interface GenerateLLMsTextParams {+ /**+ * Maximum number of URLs to process (1-100)+ * @default 10+ */+ maxUrls?: number;+ /**+ * Whether to show the full LLMs-full.txt in the response+ * @default false+ */+ showFullText?: boolean;+ /**+ * Whether to use cached content if available+ * @default true+ */+ cache?: boolean;+ /**+ * Experimental flag for streaming+ */+ __experimental_stream?: boolean;+}++/**+ * Response interface for LLMs.txt generation operations.+ */+export interface GenerateLLMsTextResponse {+ success: boolean;+ id: string;+}++/**+ * Status response interface for LLMs.txt generation operations.+ */+export interface GenerateLLMsTextStatusResponse {+ success: boolean;+ data: {+ llmstxt: string;+ llmsfulltxt?: string;+ };+ status: "processing" | "completed" | "failed";+ error?: string;+ expiresAt: string;+}++/**+ * Main class for interacting with the Firecrawl API.+ * Provides methods for scraping, searching, crawling, and mapping web content.+ */+export default class FirecrawlApp {+ public apiKey: string;+ public apiUrl: string;+ public version: string = "1.25.1";++ private isCloudService(url: string): boolean {+ return url.includes('api.firecrawl.dev');+ }++ private async getVersion(): Promise<string> {+ try {+ const packageJson = await import('../package.json', { assert: { type: 'json' } });+ return packageJson.default.version;+ } catch (error) {+ console.error("Error getting version:", error);+ return "1.25.1";+ }+ }++ private async init() {+ this.version = await this.getVersion();+ }++ /**+ * Initializes a new instance of the FirecrawlApp class.+ * @param config - Configuration options for the FirecrawlApp instance.+ */+ constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {+ const baseUrl = apiUrl || "https://api.firecrawl.dev";++ if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {+ throw new FirecrawlError("No API key provided", 401);+ }++ this.apiKey = apiKey || '';+ this.apiUrl = baseUrl;+ this.init();+ }++ /**+ * Scrapes a URL using the Firecrawl API.+ * @param url - The URL to scrape.+ * @param params - Additional parameters for the scrape request.+ * @returns The response from the scrape operation.+ */+ async scrapeUrl<T extends zt.ZodSchema, ActionsSchema extends (Action[] | undefined) = undefined>(+ url: string,+ params?: ScrapeParams<T, ActionsSchema>+ ): Promise<ScrapeResponse<zt.infer<T>, ActionsSchema extends Action[] ? ActionsResult : never> | ErrorResponse> {+ const headers: AxiosRequestHeaders = {+ "Content-Type": "application/json",+ Authorization: `Bearer ${this.apiKey}`,+ } as AxiosRequestHeaders;+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };+ if (jsonData?.extract?.schema) {+ let schema = jsonData.extract.schema;++ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {++ }+ jsonData = {+ ...jsonData,+ extract: {+ ...jsonData.extract,+ schema: schema,+ },+ };+ }++ if (jsonData?.jsonOptions?.schema) {+ let schema = jsonData.jsonOptions.schema;+ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {++ }+ jsonData = {+ ...jsonData,+ jsonOptions: {+ ...jsonData.jsonOptions,+ schema: schema,+ },+ };+ }+ try {+ const response: AxiosResponse = await axios.post(+ this.apiUrl + `/v1/scrape`,+ jsonData,+ { headers, timeout: params?.timeout !== undefined ? (params.timeout + 5000) : undefined },+ );+ if (response.status === 200) {+ const responseData = response.data;+ if (responseData.success) {+ return {+ success: true,+ warning: responseData.warning,+ error: responseData.error,+ ...responseData.data+ };+ } else {+ throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);+ }+ } else {+ this.handleError(response, "scrape URL");+ }+ } catch (error: any) {+ this.handleError(error.response, "scrape URL");+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Searches using the Firecrawl API and optionally scrapes the results.+ * @param query - The search query string.+ * @param params - Optional parameters for the search request.+ * @returns The response from the search operation.+ */+ async search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse> {+ const headers: AxiosRequestHeaders = {+ "Content-Type": "application/json",+ Authorization: `Bearer ${this.apiKey}`,+ } as AxiosRequestHeaders;++ let jsonData: any = {+ query,+ limit: params?.limit ?? 5,+ tbs: params?.tbs,+ filter: params?.filter,+ lang: params?.lang ?? "en",+ country: params?.country ?? "us",+ location: params?.location,+ origin: `js-sdk@${this.version}`,+ timeout: params?.timeout ?? 60000,+ scrapeOptions: params?.scrapeOptions ?? { formats: [] },+ };++ if (jsonData?.scrapeOptions?.extract?.schema) {+ let schema = jsonData.scrapeOptions.extract.schema;++ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {++ }+ jsonData = {+ ...jsonData,+ scrapeOptions: {+ ...jsonData.scrapeOptions,+ extract: {+ ...jsonData.scrapeOptions.extract,+ schema: schema,+ },+ },+ };+ }++ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/search`,+ jsonData,+ headers+ );++ if (response.status === 200) {+ const responseData = response.data;+ if (responseData.success) {+ return {+ success: true,+ data: responseData.data as FirecrawlDocument<any>[],+ warning: responseData.warning,+ };+ } else {+ throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);+ }+ } else {+ this.handleError(response, "search");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error.", data: [] };+ }++ /**+ * Initiates a crawl job for a URL using the Firecrawl API.+ * @param url - The URL to crawl.+ * @param params - Additional parameters for the crawl request.+ * @param pollInterval - Time in seconds for job status checks.+ * @param idempotencyKey - Optional idempotency key for the request.+ * @returns The response from the crawl operation.+ */+ async crawlUrl(+ url: string,+ params?: CrawlParams,+ pollInterval: number = 2,+ idempotencyKey?: string+ ): Promise<CrawlStatusResponse | ErrorResponse> {+ const headers = this.prepareHeaders(idempotencyKey);+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };+ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/crawl`,+ jsonData,+ headers+ );+ if (response.status === 200) {+ const id: string = response.data.id;+ return this.monitorJobStatus(id, headers, pollInterval);+ } else {+ this.handleError(response, "start crawl job");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ async asyncCrawlUrl(+ url: string,+ params?: CrawlParams,+ idempotencyKey?: string+ ): Promise<CrawlResponse | ErrorResponse> {+ const headers = this.prepareHeaders(idempotencyKey);+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };+ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/crawl`,+ jsonData,+ headers+ );+ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start crawl job");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Checks the status of a crawl job using the Firecrawl API.+ * @param id - The ID of the crawl operation.+ * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)+ * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.+ * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.+ * @param limit - How many entries to return. Only used when `getAllData = false`.+ * @returns The response containing the job status.+ */+ async checkCrawlStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse> {+ if (!id) {+ throw new FirecrawlError("No crawl ID provided", 400);+ }++ const headers: AxiosRequestHeaders = this.prepareHeaders();+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);+ if (skip !== undefined) {+ targetURL.searchParams.set("skip", skip.toString());+ }+ if (limit !== undefined) {+ targetURL.searchParams.set("limit", limit.toString());+ }++ try {+ const response: AxiosResponse = await this.getRequest(+ targetURL.href,+ headers+ );+ if (response.status === 200) {+ let allData = response.data.data;+ if (getAllData && response.data.status === "completed") {+ let statusData = response.data+ if ("data" in statusData) {+ let data = statusData.data;+ while (typeof statusData === 'object' && 'next' in statusData) {+ if (data.length === 0) {+ break+ }+ statusData = (await this.getRequest(statusData.next, headers)).data;+ data = data.concat(statusData.data);+ }+ allData = data;+ }+ }++ let resp: CrawlStatusResponse | ErrorResponse = {+ success: response.data.success,+ status: response.data.status,+ total: response.data.total,+ completed: response.data.completed,+ creditsUsed: response.data.creditsUsed,+ next: getAllData ? undefined : response.data.next,+ expiresAt: new Date(response.data.expiresAt),+ data: allData+ }++ if (!response.data.success && response.data.error) {+ resp = {+ ...resp,+ success: false,+ error: response.data.error+ } as ErrorResponse;+ }++ if (response.data.next) {+ (resp as CrawlStatusResponse).next = response.data.next;+ }++ return resp;+ } else {+ this.handleError(response, "check crawl status");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Returns information about crawl errors.+ * @param id - The ID of the crawl operation.+ * @returns Information about crawl errors.+ */+ async checkCrawlErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.deleteRequest(+ `${this.apiUrl}/v1/crawl/${id}/errors`,+ headers+ );+ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "check crawl errors");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Cancels a crawl job using the Firecrawl API.+ * @param id - The ID of the crawl operation.+ * @returns The response from the cancel crawl operation.+ */+ async cancelCrawl(id: string): Promise<ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.deleteRequest(+ `${this.apiUrl}/v1/crawl/${id}`,+ headers+ );+ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "cancel crawl job");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Initiates a crawl job and returns a CrawlWatcher to monitor the job via WebSocket.+ * @param url - The URL to crawl.+ * @param params - Additional parameters for the crawl request.+ * @param idempotencyKey - Optional idempotency key for the request.+ * @returns A CrawlWatcher instance to monitor the crawl job.+ */+ async crawlUrlAndWatch(+ url: string,+ params?: CrawlParams,+ idempotencyKey?: string,+ ) {+ const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);++ if (crawl.success && crawl.id) {+ const id = crawl.id;+ return new CrawlWatcher(id, this);+ }++ throw new FirecrawlError("Crawl job failed to start", 400);+ }++ /**+ * Maps a URL using the Firecrawl API.+ * @param url - The URL to map.+ * @param params - Additional parameters for the map request.+ * @returns The response from the map operation.+ */+ async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };++ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/map`,+ jsonData,+ headers+ );+ if (response.status === 200) {+ return response.data as MapResponse;+ } else {+ this.handleError(response, "map");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Initiates a batch scrape job for multiple URLs using the Firecrawl API.+ * @param url - The URLs to scrape.+ * @param params - Additional parameters for the scrape request.+ * @param pollInterval - Time in seconds for job status checks.+ * @param idempotencyKey - Optional idempotency key for the request.+ * @param webhook - Optional webhook for the batch scrape.+ * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.+ * @returns The response from the crawl operation.+ */+ async batchScrapeUrls(+ urls: string[],+ params?: ScrapeParams,+ pollInterval: number = 2,+ idempotencyKey?: string,+ webhook?: CrawlParams["webhook"],+ ignoreInvalidURLs?: boolean,+ maxConcurrency?: number,+ ): Promise<BatchScrapeStatusResponse | ErrorResponse> {+ const headers = this.prepareHeaders(idempotencyKey);+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: `js-sdk@${this.version}` };+ if (jsonData?.extract?.schema) {+ let schema = jsonData.extract.schema;++ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {++ }+ jsonData = {+ ...jsonData,+ extract: {+ ...jsonData.extract,+ schema: schema,+ },+ };+ }+ if (jsonData?.jsonOptions?.schema) {+ let schema = jsonData.jsonOptions.schema;++ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {++ }+ jsonData = {+ ...jsonData,+ jsonOptions: {+ ...jsonData.jsonOptions,+ schema: schema,+ },+ };+ }+ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/batch/scrape`,+ jsonData,+ headers+ );+ if (response.status === 200) {+ const id: string = response.data.id;+ return this.monitorJobStatus(id, headers, pollInterval);+ } else {+ this.handleError(response, "start batch scrape job");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ async asyncBatchScrapeUrls(+ urls: string[],+ params?: ScrapeParams,+ idempotencyKey?: string,+ webhook?: CrawlParams["webhook"],+ ignoreInvalidURLs?: boolean,+ ): Promise<BatchScrapeResponse | ErrorResponse> {+ const headers = this.prepareHeaders(idempotencyKey);+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };+ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/batch/scrape`,+ jsonData,+ headers+ );+ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start batch scrape job");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.+ * @param urls - The URL to scrape.+ * @param params - Additional parameters for the scrape request.+ * @param idempotencyKey - Optional idempotency key for the request.+ * @returns A CrawlWatcher instance to monitor the crawl job.+ */+ async batchScrapeUrlsAndWatch(+ urls: string[],+ params?: ScrapeParams,+ idempotencyKey?: string,+ webhook?: CrawlParams["webhook"],+ ignoreInvalidURLs?: boolean,+ ) {+ const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);++ if (crawl.success && crawl.id) {+ const id = crawl.id;+ return new CrawlWatcher(id, this);+ }++ throw new FirecrawlError("Batch scrape job failed to start", 400);+ }++ /**+ * Checks the status of a batch scrape job using the Firecrawl API.+ * @param id - The ID of the batch scrape operation.+ * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)+ * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.+ * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.+ * @param limit - How many entries to return. Only used when `getAllData = false`.+ * @returns The response containing the job status.+ */+ async checkBatchScrapeStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse> {+ if (!id) {+ throw new FirecrawlError("No batch scrape ID provided", 400);+ }++ const headers: AxiosRequestHeaders = this.prepareHeaders();+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);+ if (skip !== undefined) {+ targetURL.searchParams.set("skip", skip.toString());+ }+ if (limit !== undefined) {+ targetURL.searchParams.set("limit", limit.toString());+ }++ try {+ const response: AxiosResponse = await this.getRequest(+ targetURL.href,+ headers+ );+ if (response.status === 200) {+ let allData = response.data.data;+ if (getAllData && response.data.status === "completed") {+ let statusData = response.data+ if ("data" in statusData) {+ let data = statusData.data;+ while (typeof statusData === 'object' && 'next' in statusData) {+ if (data.length === 0) {+ break+ }+ statusData = (await this.getRequest(statusData.next, headers)).data;+ data = data.concat(statusData.data);+ }+ allData = data;+ }+ }++ let resp: BatchScrapeStatusResponse | ErrorResponse = {+ success: response.data.success,+ status: response.data.status,+ total: response.data.total,+ completed: response.data.completed,+ creditsUsed: response.data.creditsUsed,+ next: getAllData ? undefined : response.data.next,+ expiresAt: new Date(response.data.expiresAt),+ data: allData+ }++ if (!response.data.success && response.data.error) {+ resp = {+ ...resp,+ success: false,+ error: response.data.error+ } as ErrorResponse;+ }++ if (response.data.next) {+ (resp as BatchScrapeStatusResponse).next = response.data.next;+ }++ return resp;+ } else {+ this.handleError(response, "check batch scrape status");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Returns information about batch scrape errors.+ * @param id - The ID of the batch scrape operation.+ * @returns Information about batch scrape errors.+ */+ async checkBatchScrapeErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.deleteRequest(+ `${this.apiUrl}/v1/batch/scrape/${id}/errors`,+ headers+ );+ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "check batch scrape errors");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Extracts information from URLs using the Firecrawl API.+ * Currently in Beta. Expect breaking changes on future minor versions.+ * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.+ * @param params - Additional parameters for the extract request.+ * @returns The response from the extract operation.+ */+ async extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {+ const headers = this.prepareHeaders();++ let jsonData: { urls?: string[] } & ExtractParams<T> = { urls: urls, ...params };+ let jsonSchema: any;+ try {+ if (!params?.schema) {+ jsonSchema = undefined;+ } else {+ try {+ jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);+ } catch (_) {+ jsonSchema = params.schema;+ }+ }+ } catch (error: any) {+ throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);+ }++ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/extract`,+ { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },+ headers+ );++ if (response.status === 200) {+ const jobId = response.data.id;+ let extractStatus;+ do {+ const statusResponse: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/extract/${jobId}`,+ headers+ );+ extractStatus = statusResponse.data;+ if (extractStatus.status === "completed") {+ if (extractStatus.success) {+ return {+ success: true,+ data: extractStatus.data,+ warning: extractStatus.warning,+ error: extractStatus.error,+ sources: extractStatus?.sources || undefined,+ };+ } else {+ throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);+ }+ } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {+ throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);+ }+ await new Promise(resolve => setTimeout(resolve, 1000)); // Polling interval+ } while (extractStatus.status !== "completed");+ } else {+ this.handleError(response, "extract");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);+ }+ return { success: false, error: "Internal server error."};+ }++ /**+ * Initiates an asynchronous extract job for a URL using the Firecrawl API.+ * @param url - The URL to extract data from.+ * @param params - Additional parameters for the extract request.+ * @param idempotencyKey - Optional idempotency key for the request.+ * @returns The response from the extract operation.+ */+ async asyncExtract(+ urls: string[],+ params?: ExtractParams,+ idempotencyKey?: string+ ): Promise<ExtractResponse | ErrorResponse> {+ const headers = this.prepareHeaders(idempotencyKey);+ let jsonData: any = { urls, ...params };+ let jsonSchema: any;++ try {+ if (!params?.schema) {+ jsonSchema = undefined;+ } else {+ try {+ jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);+ } catch (_) {+ jsonSchema = params.schema;+ }+ }+ } catch (error: any) {+ throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);+ }++ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/extract`,+ { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start extract job");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Retrieves the status of an extract job.+ * @param jobId - The ID of the extract job.+ * @returns The status of the extract job.+ */+ async getExtractStatus(jobId: string): Promise<any> {+ try {+ const response: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/extract/${jobId}`,+ this.prepareHeaders()+ );++ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "get extract status");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ }++ /**+ * Prepares the headers for an API request.+ * @param idempotencyKey - Optional key to ensure idempotency.+ * @returns The prepared headers.+ */+ prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {+ return {+ "Content-Type": "application/json",+ Authorization: `Bearer ${this.apiKey}`,+ ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),+ } as AxiosRequestHeaders & { "x-idempotency-key"?: string };+ }++ /**+ * Sends a POST request to the specified URL.+ * @param url - The URL to send the request to.+ * @param data - The data to send in the request.+ * @param headers - The headers for the request.+ * @returns The response from the POST request.+ */+ postRequest(+ url: string,+ data: any,+ headers: AxiosRequestHeaders+ ): Promise<AxiosResponse> {+ return axios.post(url, data, { headers, timeout: (data?.timeout ? (data.timeout + 5000) : undefined) });+ }++ /**+ * Sends a GET request to the specified URL.+ * @param url - The URL to send the request to.+ * @param headers - The headers for the request.+ * @returns The response from the GET request.+ */+ async getRequest(+ url: string,+ headers: AxiosRequestHeaders+ ): Promise<AxiosResponse> {+ try {+ return await axios.get(url, { headers });+ } catch (error) {+ if (error instanceof AxiosError && error.response) {+ return error.response as AxiosResponse;+ } else {+ throw error;+ }+ }+ }++ /**+ * Sends a DELETE request to the specified URL.+ * @param url - The URL to send the request to.+ * @param headers - The headers for the request.+ * @returns The response from the DELETE request.+ */+ async deleteRequest(+ url: string,+ headers: AxiosRequestHeaders+ ): Promise<AxiosResponse> {+ try {+ return await axios.delete(url, { headers });+ } catch (error) {+ if (error instanceof AxiosError && error.response) {+ return error.response as AxiosResponse;+ } else {+ throw error;+ }+ }+ }++ /**+ * Monitors the status of a crawl job until completion or failure.+ * @param id - The ID of the crawl operation.+ * @param headers - The headers for the request.+ * @param checkInterval - Interval in seconds for job status checks.+ * @param checkUrl - Optional URL to check the status (used for v1 API)+ * @returns The final job status or data.+ */+ async monitorJobStatus(+ id: string,+ headers: AxiosRequestHeaders,+ checkInterval: number+ ): Promise<CrawlStatusResponse | ErrorResponse> {+ let failedTries = 0;+ let networkRetries = 0;+ const maxNetworkRetries = 3;++ while (true) {+ try {+ let statusResponse: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/crawl/${id}`,+ headers+ );++ if (statusResponse.status === 200) {+ failedTries = 0;+ networkRetries = 0;+ let statusData = statusResponse.data;++ if (statusData.status === "completed") {+ if ("data" in statusData) {+ let data = statusData.data;+ while (typeof statusData === 'object' && 'next' in statusData) {+ if (data.length === 0) {+ break+ }+ statusResponse = await this.getRequest(statusData.next, headers);+ statusData = statusResponse.data;+ data = data.concat(statusData.data);+ }+ statusData.data = data;+ return statusData;+ } else {+ throw new FirecrawlError("Crawl job completed but no data was returned", 500);+ }+ } else if (+ ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)+ ) {+ checkInterval = Math.max(checkInterval, 2);+ await new Promise((resolve) =>+ setTimeout(resolve, checkInterval * 1000)+ );+ } else {+ throw new FirecrawlError(+ `Crawl job failed or was stopped. Status: ${statusData.status}`,+ 500+ );+ }+ } else {+ failedTries++;+ if (failedTries >= 3) {+ this.handleError(statusResponse, "check crawl status");+ }+ }+ } catch (error: any) {+ if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {+ networkRetries++;+ const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);++ await new Promise((resolve) => setTimeout(resolve, backoffDelay));+ continue;+ }++ throw new FirecrawlError(error, 500);+ }+ }+ }++ /**+ * Determines if an error is retryable (transient network error)+ * @param error - The error to check+ * @returns True if the error should be retried+ */+ private isRetryableError(error: any): boolean {+ if (error instanceof AxiosError) {+ if (!error.response) {+ const code = error.code;+ const message = error.message?.toLowerCase() || '';++ return (+ code === 'ECONNRESET' ||+ code === 'ETIMEDOUT' ||+ code === 'ENOTFOUND' ||+ code === 'ECONNREFUSED' ||+ message.includes('socket hang up') ||+ message.includes('network error') ||+ message.includes('timeout')+ );+ }++ if (error.response?.status === 408 || error.response?.status === 504) {+ return true;+ }+ }++ if (error && typeof error === 'object') {+ const code = error.code;+ const message = error.message?.toLowerCase() || '';++ if (code === 'ECONNRESET' ||+ code === 'ETIMEDOUT' ||+ code === 'ENOTFOUND' ||+ code === 'ECONNREFUSED' ||+ message.includes('socket hang up') ||+ message.includes('network error') ||+ message.includes('timeout')) {+ return true;+ }++ if (error.response?.status === 408 || error.response?.status === 504) {+ return true;+ }+ }++ return false;+ }++ /**+ * Handles errors from API responses.+ * @param {AxiosResponse} response - The response from the API.+ * @param {string} action - The action being performed when the error occurred.+ */+ handleError(response: AxiosResponse, action: string): void {+ if (!response) {+ throw new FirecrawlError(+ `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,+ 0+ );+ }++ if ([400, 402, 403, 408, 409, 500].includes(response.status)) {+ const errorMessage: string =+ response.data.error || "Unknown error occurred";+ const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : '';+ throw new FirecrawlError(+ `Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}${details}`,+ response.status,+ response?.data?.details+ );+ } else {+ throw new FirecrawlError(+ `Unexpected error occurred while trying to ${action}. Status code: ${response.status}`,+ response.status+ );+ }+ }++ /**+ * Initiates a deep research operation on a given query and polls until completion.+ * @param query - The query to research.+ * @param params - Parameters for the deep research operation.+ * @param onActivity - Optional callback to receive activity updates in real-time.+ * @param onSource - Optional callback to receive source updates in real-time.+ * @returns The final research results.+ */+ async deepResearch(+ query: string,+ params: DeepResearchParams<zt.ZodSchema>,+ onActivity?: (activity: {+ type: string;+ status: string;+ message: string;+ timestamp: string;+ depth: number;+ }) => void,+ onSource?: (source: {+ url: string;+ title?: string;+ description?: string;+ icon?: string;+ }) => void+ ): Promise<DeepResearchStatusResponse | ErrorResponse> {+ try {+ const response = await this.asyncDeepResearch(query, params);++ if (!response.success || 'error' in response) {+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };+ }++ if (!response.id) {+ throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);+ }++ const jobId = response.id;+ let researchStatus;+ let lastActivityCount = 0;+ let lastSourceCount = 0;++ while (true) {+ researchStatus = await this.checkDeepResearchStatus(jobId);++ if ('error' in researchStatus && !researchStatus.success) {+ return researchStatus;+ }++ // Stream new activities through the callback if provided+ if (onActivity && researchStatus.activities) {+ const newActivities = researchStatus.activities.slice(lastActivityCount);+ for (const activity of newActivities) {+ onActivity(activity);+ }+ lastActivityCount = researchStatus.activities.length;+ }++ // Stream new sources through the callback if provided+ if (onSource && researchStatus.sources) {+ const newSources = researchStatus.sources.slice(lastSourceCount);+ for (const source of newSources) {+ onSource(source);+ }+ lastSourceCount = researchStatus.sources.length;+ }++ if (researchStatus.status === "completed") {+ return researchStatus;+ }++ if (researchStatus.status === "failed") {+ throw new FirecrawlError(+ `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,+ 500+ );+ }++ if (researchStatus.status !== "processing") {+ break;+ }++ await new Promise(resolve => setTimeout(resolve, 2000));+ }++ return { success: false, error: "Research job terminated unexpectedly" };+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);+ }+ }++ /**+ * Initiates a deep research operation on a given query without polling.+ * @param params - Parameters for the deep research operation.+ * @returns The response containing the research job ID.+ */+ async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ let jsonData: any = { query, ...params, origin: `js-sdk@${this.version}` };++ if (jsonData?.jsonOptions?.schema) {+ let schema = jsonData.jsonOptions.schema;+ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {+ // Ignore error if schema can't be parsed as Zod+ }+ jsonData = {+ ...jsonData,+ jsonOptions: {+ ...jsonData.jsonOptions,+ schema: schema,+ },+ };+ }++ try {+ const response: AxiosResponse = await this.postRequest(+ `${this.apiUrl}/v1/deep-research`,+ jsonData,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start deep research");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Checks the status of a deep research operation.+ * @param id - The ID of the deep research operation.+ * @returns The current status and results of the research operation.+ */+ async checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/deep-research/${id}`,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else if (response.status === 404) {+ throw new FirecrawlError("Deep research job not found", 404);+ } else {+ this.handleError(response, "check deep research status");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * @deprecated Use deepResearch() instead+ * Initiates a deep research operation on a given topic and polls until completion.+ * @param topic - The topic to research.+ * @param params - Parameters for the deep research operation.+ * @param onActivity - Optional callback to receive activity updates in real-time.+ * @returns The final research results.+ */+ async __deepResearch(+ topic: string,+ params: DeepResearchParams,+ onActivity?: (activity: {+ type: string;+ status: string;+ message: string;+ timestamp: string;+ depth: number;+ }) => void+ ): Promise<DeepResearchStatusResponse | ErrorResponse> {+ try {+ const response = await this.__asyncDeepResearch(topic, params);++ if (!response.success || 'error' in response) {+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };+ }++ if (!response.id) {+ throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);+ }++ const jobId = response.id;+ let researchStatus;+ let lastActivityCount = 0;++ while (true) {+ researchStatus = await this.__checkDeepResearchStatus(jobId);++ if ('error' in researchStatus && !researchStatus.success) {+ return researchStatus;+ }++ // Stream new activities through the callback if provided+ if (onActivity && researchStatus.activities) {+ const newActivities = researchStatus.activities.slice(lastActivityCount);+ for (const activity of newActivities) {+ onActivity(activity);+ }+ lastActivityCount = researchStatus.activities.length;+ }++ if (researchStatus.status === "completed") {+ return researchStatus;+ }++ if (researchStatus.status === "failed") {+ throw new FirecrawlError(+ `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,+ 500+ );+ }++ if (researchStatus.status !== "processing") {+ break;+ }++ await new Promise(resolve => setTimeout(resolve, 2000));+ }++ return { success: false, error: "Research job terminated unexpectedly" };+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);+ }+ }++ /**+ * @deprecated Use asyncDeepResearch() instead+ * Initiates a deep research operation on a given topic without polling.+ * @param params - Parameters for the deep research operation.+ * @returns The response containing the research job ID.+ */+ async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ let jsonData: any = { topic, ...params, origin: `js-sdk@${this.version}` };+ const response: AxiosResponse = await this.postRequest(+ `${this.apiUrl}/v1/deep-research`,+ jsonData,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start deep research");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * @deprecated Use checkDeepResearchStatus() instead+ * Checks the status of a deep research operation.+ * @param id - The ID of the deep research operation.+ * @returns The current status and results of the research operation.+ */+ async __checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/deep-research/${id}`,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else if (response.status === 404) {+ throw new FirecrawlError("Deep research job not found", 404);+ } else {+ this.handleError(response, "check deep research status");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Generates LLMs.txt for a given URL and polls until completion.+ * @param url - The URL to generate LLMs.txt from.+ * @param params - Parameters for the LLMs.txt generation operation.+ * @returns The final generation results.+ */+ async generateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {+ try {+ const response = await this.asyncGenerateLLMsText(url, params);++ if (!response.success || 'error' in response) {+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };+ }++ if (!response.id) {+ throw new FirecrawlError(`Failed to start LLMs.txt generation. No job ID returned.`, 500);+ }++ const jobId = response.id;+ let generationStatus;++ while (true) {+ generationStatus = await this.checkGenerateLLMsTextStatus(jobId);++ if ('error' in generationStatus && !generationStatus.success) {+ return generationStatus;+ }++ if (generationStatus.status === "completed") {+ return generationStatus;+ }++ if (generationStatus.status === "failed") {+ throw new FirecrawlError(+ `LLMs.txt generation job ${generationStatus.status}. Error: ${generationStatus.error}`,+ 500+ );+ }++ if (generationStatus.status !== "processing") {+ break;+ }++ await new Promise(resolve => setTimeout(resolve, 2000));+ }++ return { success: false, error: "LLMs.txt generation job terminated unexpectedly" };+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);+ }+ }++ /**+ * Initiates a LLMs.txt generation operation without polling.+ * @param url - The URL to generate LLMs.txt from.+ * @param params - Parameters for the LLMs.txt generation operation.+ * @returns The response containing the generation job ID.+ */+ async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };+ try {+ const response: AxiosResponse = await this.postRequest(+ `${this.apiUrl}/v1/llmstxt`,+ jsonData,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start LLMs.txt generation");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Checks the status of a LLMs.txt generation operation.+ * @param id - The ID of the LLMs.txt generation operation.+ * @returns The current status and results of the generation operation.+ */+ async checkGenerateLLMsTextStatus(id: string): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/llmstxt/${id}`,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else if (response.status === 404) {+ throw new FirecrawlError("LLMs.txt generation job not found", 404);+ } else {+ this.handleError(response, "check LLMs.txt generation status");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }+}++interface CrawlWatcherEvents {+ document: CustomEvent<FirecrawlDocument<undefined>>,+ done: CustomEvent<{+ status: CrawlStatusResponse["status"];+ data: FirecrawlDocument<undefined>[];+ }>,+ error: CustomEvent<{+ status: CrawlStatusResponse["status"],+ data: FirecrawlDocument<undefined>[],+ error: string,+ }>,+}++export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {+ private ws: WebSocket;+ public data: FirecrawlDocument<undefined>[];+ public status: CrawlStatusResponse["status"];+ public id: string;++ constructor(id: string, app: FirecrawlApp) {+ super();+ this.id = id;+ // replace `http` with `ws` (`http://` -> `ws://` and `https://` -> `wss://`)+ const wsUrl = app.apiUrl.replace(/^http/, "ws");+ this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);+ this.status = "scraping";+ this.data = [];++ type ErrorMessage = {+ type: "error",+ error: string,+ }++ type CatchupMessage = {+ type: "catchup",+ data: CrawlStatusResponse,+ }++ type DocumentMessage = {+ type: "document",+ data: FirecrawlDocument<undefined>,+ }++ type DoneMessage = { type: "done" }++ type Message = ErrorMessage | CatchupMessage | DoneMessage | DocumentMessage;++ const messageHandler = (msg: Message) => {+ if (msg.type === "done") {+ this.status = "completed";+ this.dispatchTypedEvent("done", new CustomEvent("done", {+ detail: {+ status: this.status,+ data: this.data,+ id: this.id,+ },+ }));+ } else if (msg.type === "error") {+ this.status = "failed";+ this.dispatchTypedEvent("error", new CustomEvent("error", {+ detail: {+ status: this.status,+ data: this.data,+ error: msg.error,+ id: this.id,+ },+ }));+ } else if (msg.type === "catchup") {+ this.status = msg.data.status;+ this.data.push(...(msg.data.data ?? []));+ for (const doc of this.data) {+ this.dispatchTypedEvent("document", new CustomEvent("document", {+ detail: {+ ...doc,+ id: this.id,+ },+ }));+ }+ } else if (msg.type === "document") {+ this.dispatchTypedEvent("document", new CustomEvent("document", {+ detail: {+ ...msg.data,+ id: this.id,+ },+ }));+ }+ }++ this.ws.onmessage = ((ev: MessageEvent) => {+ if (typeof ev.data !== "string") {+ this.ws.close();+ return;+ }+ try {+ const msg = JSON.parse(ev.data) as Message;+ messageHandler(msg);+ } catch (error) {+ console.error("Error on message", error);+ }+ }).bind(this);++ this.ws.onclose = ((ev: CloseEvent) => {+ try {+ const msg = JSON.parse(ev.reason) as Message;+ messageHandler(msg);+ } catch (error) {+ console.error("Error on close", error);+ }+ }).bind(this);++ this.ws.onerror = ((_: Event) => {+ this.status = "failed"+ this.dispatchTypedEvent("error", new CustomEvent("error", {+ detail: {+ status: this.status,+ data: this.data,+ error: "WebSocket error",+ id: this.id,+ },+ }));+ }).bind(this);+ }++ close() {+ this.ws.close();+ }+}diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex f93bb48e3..e068a269c 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -1,2146 +1,25 @@-import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios";-import * as zt from "zod";-import { zodToJsonSchema } from "zod-to-json-schema";-import { TypedEventTarget } from "typescript-event-target";+// Unified entrypoint for Firecrawl JS SDK+// - Default export: unified Firecrawl (v2 by default, v1 under .v1)+// - Named export FirecrawlClient: direct v2 client+// - Named export FirecrawlAppV1: legacy v1 class+// - Re-export v2 types-/**- * Configuration interface for FirecrawlApp.- * @param apiKey - Optional API key for authentication.- * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.- */-export interface FirecrawlAppConfig {- apiKey?: string | null;- apiUrl?: string | null;-}--/**- * Metadata for a Firecrawl document.- * Includes various optional properties for document metadata.- */-export interface FirecrawlDocumentMetadata {- title?: string;- description?: string;- language?: string;- keywords?: string;- robots?: string;- ogTitle?: string;- ogDescription?: string;- ogUrl?: string;- ogImage?: string;- ogAudio?: string;- ogDeterminer?: string;- ogLocale?: string;- ogLocaleAlternate?: string[];- ogSiteName?: string;- ogVideo?: string;- dctermsCreated?: string;- dcDateCreated?: string;- dcDate?: string;- dctermsType?: string;- dcType?: string;- dctermsAudience?: string;- dctermsSubject?: string;- dcSubject?: string;- dcDescription?: string;- dctermsKeywords?: string;- modifiedTime?: string;- publishedTime?: string;- articleTag?: string;- articleSection?: string;- sourceURL?: string;- statusCode?: number;- error?: string;- proxyUsed?: "basic" | "stealth";- cacheState?: "miss" | "hit";- cachedAt?: string;- [key: string]: any; // Allows for additional metadata properties not explicitly defined.-}--/**- * Document interface for Firecrawl.- * Represents a document retrieved or processed by Firecrawl.- */-export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | never) = never> {- url?: string;- markdown?: string;- html?: string;- rawHtml?: string;- links?: string[];- extract?: T;- json?: T;- screenshot?: string;- metadata?: FirecrawlDocumentMetadata;- actions: ActionsSchema;- changeTracking?: {- previousScrapeAt: string | null;- changeStatus: "new" | "same" | "changed" | "removed";- visibility: "visible" | "hidden";- diff?: {- text: string;- json: {- files: Array<{- from: string | null;- to: string | null;- chunks: Array<{- content: string;- changes: Array<{- type: string;- normal?: boolean;- ln?: number;- ln1?: number;- ln2?: number;- content: string;- }>;- }>;- }>;- };- };- json?: any;- };- // v1 search only- title?: string;- description?: string;-}--/**- * Parameters for scraping operations.- * Defines the options and configurations available for scraping web content.- */-export interface CrawlScrapeOptions {- formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "changeTracking")[];- headers?: Record<string, string>;- includeTags?: string[];- excludeTags?: string[];- onlyMainContent?: boolean;- waitFor?: number;- timeout?: number;- location?: {- country?: string;- languages?: string[];- };- mobile?: boolean;- skipTlsVerification?: boolean;- removeBase64Images?: boolean;- blockAds?: boolean;- proxy?: "basic" | "stealth" | "auto";- storeInCache?: boolean;- maxAge?: number;- parsePDF?: boolean;-}--export type Action = {- type: "wait",- milliseconds?: number,- selector?: string,-} | {- type: "click",- selector: string,- all?: boolean,-} | {- type: "screenshot",- fullPage?: boolean,- quality?: number,-} | {- type: "write",- text: string,-} | {- type: "press",- key: string,-} | {- type: "scroll",- direction?: "up" | "down",- selector?: string,-} | {- type: "scrape",-} | {- type: "executeJavascript",- script: string,-};--export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema extends (Action[] | undefined) = undefined> extends CrawlScrapeOptions {- extract?: {- prompt?: string;- schema?: LLMSchema;- systemPrompt?: string;- };- jsonOptions?:{- prompt?: string;- schema?: LLMSchema;- systemPrompt?: string;- }- changeTrackingOptions?: {- prompt?: string;- schema?: any;- modes?: ("json" | "git-diff")[];- tag?: string | null;- }- actions?: ActionsSchema;- agent?: AgentOptions;- zeroDataRetention?: boolean;-}--export interface ActionsResult {- screenshots: string[];- scrapes: ({- url: string;- html: string;- })[];- javascriptReturns: {- type: string;- value: unknown- }[];-}--/**- * Response interface for scraping operations.- * Defines the structure of the response received after a scraping operation.- */-export interface ScrapeResponse<LLMResult = any, ActionsSchema extends (ActionsResult | never) = never> extends FirecrawlDocument<LLMResult, ActionsSchema> {- success: true;- warning?: string;- error?: string;-}--/**- * Parameters for crawling operations.- * Includes options for both scraping and mapping during a crawl.- */-export interface CrawlParams {- includePaths?: string[];- excludePaths?: string[];- maxDepth?: number;- maxDiscoveryDepth?: number;- limit?: number;- allowBackwardLinks?: boolean;- crawlEntireDomain?: boolean;- allowExternalLinks?: boolean;- ignoreSitemap?: boolean;- scrapeOptions?: CrawlScrapeOptions;- webhook?: string | {- url: string;- headers?: Record<string, string>;- metadata?: Record<string, string>;- events?: ["completed", "failed", "page", "started"][number][];- };- deduplicateSimilarURLs?: boolean;- ignoreQueryParameters?: boolean;- regexOnFullURL?: boolean;- /**- * Delay in seconds between scrapes. This helps respect website rate limits.- * If not provided, the crawler may use the robots.txt crawl delay if available.- */- delay?: number;- allowSubdomains?: boolean;- maxConcurrency?: number;- zeroDataRetention?: boolean;-}--/**- * Response interface for crawling operations.- * Defines the structure of the response received after initiating a crawl.- */-export interface CrawlResponse {- id?: string;- url?: string;- success: true;- error?: string;-}--/**- * Response interface for batch scrape operations.- * Defines the structure of the response received after initiating a crawl.- */-export interface BatchScrapeResponse {- id?: string;- url?: string;- success: true;- error?: string;- invalidURLs?: string[];-}--/**- * Response interface for job status checks.- * Provides detailed status of a crawl job including progress and results.- */-export interface CrawlStatusResponse {- success: true;- status: "scraping" | "completed" | "failed" | "cancelled";- completed: number;- total: number;- creditsUsed: number;- expiresAt: Date;- next?: string;- data: FirecrawlDocument<undefined>[];-};--/**- * Response interface for batch scrape job status checks.- * Provides detailed status of a batch scrape job including progress and results.- */-export interface BatchScrapeStatusResponse {- success: true;- status: "scraping" | "completed" | "failed" | "cancelled";- completed: number;- total: number;- creditsUsed: number;- expiresAt: Date;- next?: string;- data: FirecrawlDocument<undefined>[];-};--/**- * Parameters for mapping operations.- * Defines options for mapping URLs during a crawl.- */-export interface MapParams {- search?: string;- ignoreSitemap?: boolean;- includeSubdomains?: boolean;- sitemapOnly?: boolean;- limit?: number;- timeout?: number;- useIndex?: boolean;-}+export { FirecrawlClient } from "./v2/client";+export * from "./v2/types";+export { default as FirecrawlAppV1 } from "./v1";-/**- * Response interface for mapping operations.- * Defines the structure of the response received after a mapping operation.- */-export interface MapResponse {- success: true;- links?: string[];- error?: string;-}--/**- * Parameters for extracting information from URLs.- * Defines options for extracting information from URLs.- */-export interface AgentOptions {- model?: string;- prompt?: string;- sessionId?: string;-}+import V1 from "./v1";+import { FirecrawlClient as V2 } from "./v2/client";+import type { FirecrawlAppConfig } from "./v1";-/**- * Parameters for extracting information from URLs.- * Defines options for extracting information from URLs.- */-export interface AgentOptionsExtract {- model?: string;- sessionId?: string;-}+export class Firecrawl extends V2 {+ public v1: V1;-export interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {- prompt?: string;- schema?: LLMSchema | object;- systemPrompt?: string;- allowExternalLinks?: boolean;- enableWebSearch?: boolean;- includeSubdomains?: boolean;- origin?: string;- showSources?: boolean;- scrapeOptions?: CrawlScrapeOptions;- agent?: AgentOptionsExtract;-}--/**- * Response interface for extracting information from URLs.- * Defines the structure of the response received after extracting information from URLs.- */-export interface ExtractResponse<LLMSchema extends zt.ZodSchema = any> {- success: boolean;- data: LLMSchema;- error?: string;- warning?: string;- sources?: string[];-}--/**- * Error response interface.- * Defines the structure of the response received when an error occurs.- */-export interface ErrorResponse {- success: false;- error: string;-}--/**- * Custom error class for Firecrawl.- * Extends the built-in Error class to include a status code.- */-export class FirecrawlError extends Error {- statusCode: number;- details?: any;- constructor(message: string, statusCode: number, details?: any) {- super(message);- this.statusCode = statusCode;- this.details = details;+ constructor(opts: FirecrawlAppConfig = {}) {+ super(opts as any);+ this.v1 = new V1(opts);}}-/**- * Parameters for search operations.- * Defines options for searching and scraping search results.- */-export interface SearchParams {- limit?: number;- tbs?: string;- filter?: string;- lang?: string;- country?: string;- location?: string;- origin?: string;- timeout?: number;- scrapeOptions?: ScrapeParams;-}--/**- * Response interface for search operations.- * Defines the structure of the response received after a search operation.- */-export interface SearchResponse {- success: boolean;- data: FirecrawlDocument<undefined>[];- warning?: string;- error?: string;-}--/**- * Response interface for crawl/batch scrape error monitoring.- */-export interface CrawlErrorsResponse {- /**- * Scrapes that errored out + error details- */- errors: {- id: string,- timestamp?: string,- url: string,- error: string,- }[];-- /**- * URLs blocked by robots.txt- */- robotsBlocked: string[];-};--/**- * Parameters for deep research operations.- * Defines options for conducting deep research on a query.- */-export interface DeepResearchParams<LLMSchema extends zt.ZodSchema = any> {- /**- * Maximum depth of research iterations (1-10)- * @default 7- */- maxDepth?: number;- /**- * Time limit in seconds (30-300)- * @default 270- */- timeLimit?: number;- /**- * Maximum number of URLs to analyze (1-1000)- * @default 20- */- maxUrls?: number;- /**- * The prompt to use for the final analysis- */- analysisPrompt?: string;- /**- * The system prompt to use for the research agent- */- systemPrompt?: string;- /**- * The formats to use for the final analysis- */- formats?: ("markdown" | "json")[];- /**- * The JSON options to use for the final analysis- */- jsonOptions?:{- prompt?: string;- schema?: LLMSchema;- systemPrompt?: string;- };- /**- * Experimental flag for streaming steps- */- // __experimental_streamSteps?: boolean;-}--/**- * Response interface for deep research operations.- */-export interface DeepResearchResponse {- success: boolean;- id: string;-}--/**- * Status response interface for deep research operations.- */-export interface DeepResearchStatusResponse {- success: boolean;- data: {- finalAnalysis: string;- activities: Array<{- type: string;- status: string;- message: string;- timestamp: string;- depth: number;- }>;- sources: Array<{- url: string;- title: string;- description: string;- }>;- };- status: "processing" | "completed" | "failed";- error?: string;- expiresAt: string;- currentDepth: number;- maxDepth: number;- activities: Array<{- type: string;- status: string;- message: string;- timestamp: string;- depth: number;- }>;- sources: Array<{- url: string;- title: string;- description: string;- }>;- summaries: string[];-}--/**- * Parameters for LLMs.txt generation operations.- */-export interface GenerateLLMsTextParams {- /**- * Maximum number of URLs to process (1-100)- * @default 10- */- maxUrls?: number;- /**- * Whether to show the full LLMs-full.txt in the response- * @default false- */- showFullText?: boolean;- /**- * Whether to use cached content if available- * @default true- */- cache?: boolean;- /**- * Experimental flag for streaming- */- __experimental_stream?: boolean;-}--/**- * Response interface for LLMs.txt generation operations.- */-export interface GenerateLLMsTextResponse {- success: boolean;- id: string;-}--/**- * Status response interface for LLMs.txt generation operations.- */-export interface GenerateLLMsTextStatusResponse {- success: boolean;- data: {- llmstxt: string;- llmsfulltxt?: string;- };- status: "processing" | "completed" | "failed";- error?: string;- expiresAt: string;-}--/**- * Main class for interacting with the Firecrawl API.- * Provides methods for scraping, searching, crawling, and mapping web content.- */-export default class FirecrawlApp {- public apiKey: string;- public apiUrl: string;- public version: string = "1.25.1";-- private isCloudService(url: string): boolean {- return url.includes('api.firecrawl.dev');- }-- private async getVersion(): Promise<string> {- try {- const packageJson = await import('../package.json', { assert: { type: 'json' } });- return packageJson.default.version;- } catch (error) {- console.error("Error getting version:", error);- return "1.25.1";- }- }-- private async init() {- this.version = await this.getVersion();- }-- /**- * Initializes a new instance of the FirecrawlApp class.- * @param config - Configuration options for the FirecrawlApp instance.- */- constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {- const baseUrl = apiUrl || "https://api.firecrawl.dev";-- if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {- throw new FirecrawlError("No API key provided", 401);- }-- this.apiKey = apiKey || '';- this.apiUrl = baseUrl;- this.init();- }-- /**- * Scrapes a URL using the Firecrawl API.- * @param url - The URL to scrape.- * @param params - Additional parameters for the scrape request.- * @returns The response from the scrape operation.- */- async scrapeUrl<T extends zt.ZodSchema, ActionsSchema extends (Action[] | undefined) = undefined>(- url: string,- params?: ScrapeParams<T, ActionsSchema>- ): Promise<ScrapeResponse<zt.infer<T>, ActionsSchema extends Action[] ? ActionsResult : never> | ErrorResponse> {- const headers: AxiosRequestHeaders = {- "Content-Type": "application/json",- Authorization: `Bearer ${this.apiKey}`,- } as AxiosRequestHeaders;- let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };- if (jsonData?.extract?.schema) {- let schema = jsonData.extract.schema;-- // Try parsing the schema as a Zod schema- try {- schema = zodToJsonSchema(schema);- } catch (error) {-- }- jsonData = {- ...jsonData,- extract: {- ...jsonData.extract,- schema: schema,- },- };- }-- if (jsonData?.jsonOptions?.schema) {- let schema = jsonData.jsonOptions.schema;- // Try parsing the schema as a Zod schema- try {- schema = zodToJsonSchema(schema);- } catch (error) {-- }- jsonData = {- ...jsonData,- jsonOptions: {- ...jsonData.jsonOptions,- schema: schema,- },- };- }- try {- const response: AxiosResponse = await axios.post(- this.apiUrl + `/v1/scrape`,- jsonData,- { headers, timeout: params?.timeout !== undefined ? (params.timeout + 5000) : undefined },- );- if (response.status === 200) {- const responseData = response.data;- if (responseData.success) {- return {- success: true,- warning: responseData.warning,- error: responseData.error,- ...responseData.data- };- } else {- throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);- }- } else {- this.handleError(response, "scrape URL");- }- } catch (error: any) {- this.handleError(error.response, "scrape URL");- }- return { success: false, error: "Internal server error." };- }-- /**- * Searches using the Firecrawl API and optionally scrapes the results.- * @param query - The search query string.- * @param params - Optional parameters for the search request.- * @returns The response from the search operation.- */- async search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse> {- const headers: AxiosRequestHeaders = {- "Content-Type": "application/json",- Authorization: `Bearer ${this.apiKey}`,- } as AxiosRequestHeaders;-- let jsonData: any = {- query,- limit: params?.limit ?? 5,- tbs: params?.tbs,- filter: params?.filter,- lang: params?.lang ?? "en",- country: params?.country ?? "us",- location: params?.location,- origin: `js-sdk@${this.version}`,- timeout: params?.timeout ?? 60000,- scrapeOptions: params?.scrapeOptions ?? { formats: [] },- };-- if (jsonData?.scrapeOptions?.extract?.schema) {- let schema = jsonData.scrapeOptions.extract.schema;-- // Try parsing the schema as a Zod schema- try {- schema = zodToJsonSchema(schema);- } catch (error) {-- }- jsonData = {- ...jsonData,- scrapeOptions: {- ...jsonData.scrapeOptions,- extract: {- ...jsonData.scrapeOptions.extract,- schema: schema,- },- },- };- }-- try {- const response: AxiosResponse = await this.postRequest(- this.apiUrl + `/v1/search`,- jsonData,- headers- );-- if (response.status === 200) {- const responseData = response.data;- if (responseData.success) {- return {- success: true,- data: responseData.data as FirecrawlDocument<any>[],- warning: responseData.warning,- };- } else {- throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);- }- } else {- this.handleError(response, "search");- }- } catch (error: any) {- if (error.response?.data?.error) {- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);- } else {- throw new FirecrawlError(error.message, 500);- }- }- return { success: false, error: "Internal server error.", data: [] };- }-- /**- * Initiates a crawl job for a URL using the Firecrawl API.- * @param url - The URL to crawl.- * @param params - Additional parameters for the crawl request.- * @param pollInterval - Time in seconds for job status checks.- * @param idempotencyKey - Optional idempotency key for the request.- * @returns The response from the crawl operation.- */- async crawlUrl(- url: string,- params?: CrawlParams,- pollInterval: number = 2,- idempotencyKey?: string- ): Promise<CrawlStatusResponse | ErrorResponse> {- const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };- try {- const response: AxiosResponse = await this.postRequest(- this.apiUrl + `/v1/crawl`,- jsonData,- headers- );- if (response.status === 200) {- const id: string = response.data.id;- return this.monitorJobStatus(id, headers, pollInterval);- } else {- this.handleError(response, "start crawl job");- }- } catch (error: any) {- if (error.response?.data?.error) {- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);- } else {- throw new FirecrawlError(error.message, 500);- }- }- return { success: false, error: "Internal server error." };- }-- async asyncCrawlUrl(- url: string,- params?: CrawlParams,- idempotencyKey?: string- ): Promise<CrawlResponse | ErrorResponse> {- const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };- try {- const response: AxiosResponse = await this.postRequest(- this.apiUrl + `/v1/crawl`,- jsonData,- headers- );- if (response.status === 200) {- return response.data;- } else {- this.handleError(response, "start crawl job");- }- } catch (error: any) {- if (error.response?.data?.error) {- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);- } else {- throw new FirecrawlError(error.message, 500);- }- }- return { success: false, error: "Internal server error." };- }-- /**- * Checks the status of a crawl job using the Firecrawl API.- * @param id - The ID of the crawl operation.- * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)- * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.- * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.- * @param limit - How many entries to return. Only used when `getAllData = false`.- * @returns The response containing the job status.- */- async checkCrawlStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse> {- if (!id) {- throw new FirecrawlError("No crawl ID provided", 400);- }+export default Firecrawl;- const headers: AxiosRequestHeaders = this.prepareHeaders();- const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);- if (skip !== undefined) {- targetURL.searchParams.set("skip", skip.toString());- }- if (limit !== undefined) {- targetURL.searchParams.set("limit", limit.toString());- }-- try {- const response: AxiosResponse = await this.getRequest(- targetURL.href,- headers- );- if (response.status === 200) {- let allData = response.data.data;- if (getAllData && response.data.status === "completed") {- let statusData = response.data- if ("data" in statusData) {- let data = statusData.data;- while (typeof statusData === 'object' && 'next' in statusData) {- if (data.length === 0) {- break- }- statusData = (await this.getRequest(statusData.next, headers)).data;- data = data.concat(statusData.data);- }- allData = data;- }- }-- let resp: CrawlStatusResponse | ErrorResponse = {- success: response.data.success,- status: response.data.status,- total: response.data.total,- completed: response.data.completed,- creditsUsed: response.data.creditsUsed,- next: getAllData ? undefined : response.data.next,- expiresAt: new Date(response.data.expiresAt),- data: allData- }-- if (!response.data.success && response.data.error) {- resp = {- ...resp,- success: false,- error: response.data.error- } as ErrorResponse;- }-- if (response.data.next) {- (resp as CrawlStatusResponse).next = response.data.next;- }-- return resp;- } else {- this.handleError(response, "check crawl status");- }- } catch (error: any) {- throw new FirecrawlError(error.message, 500);- }- return { success: false, error: "Internal server error." };- }-- /**- * Returns information about crawl errors.- * @param id - The ID of the crawl operation.- * @returns Information about crawl errors.- */- async checkCrawlErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {- const headers = this.prepareHeaders();- try {- const response: AxiosResponse = await this.deleteRequest(- `${this.apiUrl}/v1/crawl/${id}/errors`,- headers- );- if (response.status === 200) {- return response.data;- } else {- this.handleError(response, "check crawl errors");- }- } catch (error: any) {- throw new FirecrawlError(error.message, 500);- }- return { success: false, error: "Internal server error." };- }-- /**- * Cancels a crawl job using the Firecrawl API.- * @param id - The ID of the crawl operation.- * @returns The response from the cancel crawl operation.- */- async cancelCrawl(id: string): Promise<ErrorResponse> {- const headers = this.prepareHeaders();- try {- const response: AxiosResponse = await this.deleteRequest(- `${this.apiUrl}/v1/crawl/${id}`,- headers- );- if (response.status === 200) {- return response.data;- } else {- this.handleError(response, "cancel crawl job");- }- } catch (error: any) {- throw new FirecrawlError(error.message, 500);- }- return { success: false, error: "Internal server error." };- }-- /**- * Initiates a crawl job and returns a CrawlWatcher to monitor the job via WebSocket.- * @param url - The URL to crawl.- * @param params - Additional parameters for the crawl request.- * @param idempotencyKey - Optional idempotency key for the request.- * @returns A CrawlWatcher instance to monitor the crawl job.- */- async crawlUrlAndWatch(- url: string,- params?: CrawlParams,- idempotencyKey?: string,- ) {- const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);-- if (crawl.success && crawl.id) {- const id = crawl.id;- return new CrawlWatcher(id, this);- }-- throw new FirecrawlError("Crawl job failed to start", 400);- }-- /**- * Maps a URL using the Firecrawl API.- * @param url - The URL to map.- * @param params - Additional parameters for the map request.- * @returns The response from the map operation.- */- async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {- const headers = this.prepareHeaders();- let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };-- try {- const response: AxiosResponse = await this.postRequest(- this.apiUrl + `/v1/map`,- jsonData,- headers- );- if (response.status === 200) {- return response.data as MapResponse;- } else {- this.handleError(response, "map");- }- } catch (error: any) {- throw new FirecrawlError(error.message, 500);- }- return { success: false, error: "Internal server error." };- }-- /**- * Initiates a batch scrape job for multiple URLs using the Firecrawl API.- * @param url - The URLs to scrape.- * @param params - Additional parameters for the scrape request.- * @param pollInterval - Time in seconds for job status checks.- * @param idempotencyKey - Optional idempotency key for the request.- * @param webhook - Optional webhook for the batch scrape.- * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.- * @returns The response from the crawl operation.- */- async batchScrapeUrls(- urls: string[],- params?: ScrapeParams,- pollInterval: number = 2,- idempotencyKey?: string,- webhook?: CrawlParams["webhook"],- ignoreInvalidURLs?: boolean,- maxConcurrency?: number,- ): Promise<BatchScrapeStatusResponse | ErrorResponse> {- const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: `js-sdk@${this.version}` };- if (jsonData?.extract?.schema) {- let schema = jsonData.extract.schema;-- // Try parsing the schema as a Zod schema- try {- schema = zodToJsonSchema(schema);- } catch (error) {-- }- jsonData = {- ...jsonData,- extract: {- ...jsonData.extract,- schema: schema,- },- };- }- if (jsonData?.jsonOptions?.schema) {- let schema = jsonData.jsonOptions.schema;-- // Try parsing the schema as a Zod schema- try {- schema = zodToJsonSchema(schema);- } catch (error) {-- }- jsonData = {- ...jsonData,- jsonOptions: {- ...jsonData.jsonOptions,- schema: schema,- },- };- }- try {- const response: AxiosResponse = await this.postRequest(- this.apiUrl + `/v1/batch/scrape`,- jsonData,- headers- );- if (response.status === 200) {- const id: string = response.data.id;- return this.monitorJobStatus(id, headers, pollInterval);- } else {- this.handleError(response, "start batch scrape job");- }- } catch (error: any) {- if (error.response?.data?.error) {- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);- } else {- throw new FirecrawlError(error.message, 500);- }- }- return { success: false, error: "Internal server error." };- }-- async asyncBatchScrapeUrls(- urls: string[],- params?: ScrapeParams,- idempotencyKey?: string,- webhook?: CrawlParams["webhook"],- ignoreInvalidURLs?: boolean,- ): Promise<BatchScrapeResponse | ErrorResponse> {- const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };- try {- const response: AxiosResponse = await this.postRequest(- this.apiUrl + `/v1/batch/scrape`,- jsonData,- headers- );- if (response.status === 200) {- return response.data;- } else {- this.handleError(response, "start batch scrape job");- }- } catch (error: any) {- if (error.response?.data?.error) {- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);- } else {- throw new FirecrawlError(error.message, 500);- }- }- return { success: false, error: "Internal server error." };- }-- /**- * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.- * @param urls - The URL to scrape.- * @param params - Additional parameters for the scrape request.- * @param idempotencyKey - Optional idempotency key for the request.- * @returns A CrawlWatcher instance to monitor the crawl job.- */- async batchScrapeUrlsAndWatch(- urls: string[],- params?: ScrapeParams,- idempotencyKey?: string,- webhook?: CrawlParams["webhook"],- ignoreInvalidURLs?: boolean,- ) {- const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);-- if (crawl.success && crawl.id) {- const id = crawl.id;- return new CrawlWatcher(id, this);- }-- throw new FirecrawlError("Batch scrape job failed to start", 400);- }-- /**- * Checks the status of a batch scrape job using the Firecrawl API.- * @param id - The ID of the batch scrape operation.- * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)- * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.- * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.- * @param limit - How many entries to return. Only used when `getAllData = false`.- * @returns The response containing the job status.- */- async checkBatchScrapeStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse> {- if (!id) {- throw new FirecrawlError("No batch scrape ID provided", 400);- }-- const headers: AxiosRequestHeaders = this.prepareHeaders();- const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);- if (skip !== undefined) {- targetURL.searchParams.set("skip", skip.toString());- }- if (limit !== undefined) {- targetURL.searchParams.set("limit", limit.toString());- }-- try {- const response: AxiosResponse = await this.getRequest(- targetURL.href,- headers- );- if (response.status === 200) {- let allData = response.data.data;- if (getAllData && response.data.status === "completed") {- let statusData = response.data- if ("data" in statusData) {- let data = statusData.data;- while (typeof statusData === 'object' && 'next' in statusData) {- if (data.length === 0) {- break- }- statusData = (await this.getRequest(statusData.next, headers)).data;- data = data.concat(statusData.data);- }- allData = data;- }- }-- let resp: BatchScrapeStatusResponse | ErrorResponse = {- success: response.data.success,- status: response.data.status,- total: response.data.total,- completed: response.data.completed,- creditsUsed: response.data.creditsUsed,- next: getAllData ? undefined : response.data.next,- expiresAt: new Date(response.data.expiresAt),- data: allData- }-- if (!response.data.success && response.data.error) {- resp = {- ...resp,- success: false,- error: response.data.error- } as ErrorResponse;- }-- if (response.data.next) {- (resp as BatchScrapeStatusResponse).next = response.data.next;- }-- return resp;- } else {- this.handleError(response, "check batch scrape status");- }- } catch (error: any) {- throw new FirecrawlError(error.message, 500);- }- return { success: false, error: "Internal server error." };- }-- /**- * Returns information about batch scrape errors.- * @param id - The ID of the batch scrape operation.- * @returns Information about batch scrape errors.- */- async checkBatchScrapeErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {- const headers = this.prepareHeaders();- try {- const response: AxiosResponse = await this.deleteRequest(- `${this.apiUrl}/v1/batch/scrape/${id}/errors`,- headers- );- if (response.status === 200) {- return response.data;- } else {- this.handleError(response, "check batch scrape errors");- }- } catch (error: any) {- throw new FirecrawlError(error.message, 500);- }- return { success: false, error: "Internal server error." };- }-- /**- * Extracts information from URLs using the Firecrawl API.- * Currently in Beta. Expect breaking changes on future minor versions.- * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.- * @param params - Additional parameters for the extract request.- * @returns The response from the extract operation.- */- async extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {- const headers = this.prepareHeaders();-- let jsonData: { urls?: string[] } & ExtractParams<T> = { urls: urls, ...params };- let jsonSchema: any;- try {- if (!params?.schema) {- jsonSchema = undefined;- } else {- try {- jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);- } catch (_) {- jsonSchema = params.schema;- }- }- } catch (error: any) {- throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);- }-- try {- const response: AxiosResponse = await this.postRequest(- this.apiUrl + `/v1/extract`,- { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },- headers- );-- if (response.status === 200) {- const jobId = response.data.id;- let extractStatus;- do {- const statusResponse: AxiosResponse = await this.getRequest(- `${this.apiUrl}/v1/extract/${jobId}`,- headers- );- extractStatus = statusResponse.data;- if (extractStatus.status === "completed") {- if (extractStatus.success) {- return {- success: true,- data: extractStatus.data,- warning: extractStatus.warning,- error: extractStatus.error,- sources: extractStatus?.sources || undefined,- };- } else {- throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);- }- } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {- throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);- }- await new Promise(resolve => setTimeout(resolve, 1000)); // Polling interval- } while (extractStatus.status !== "completed");- } else {- this.handleError(response, "extract");- }- } catch (error: any) {- throw new FirecrawlError(error.message, 500, error.response?.data?.details);- }- return { success: false, error: "Internal server error."};- }-- /**- * Initiates an asynchronous extract job for a URL using the Firecrawl API.- * @param url - The URL to extract data from.- * @param params - Additional parameters for the extract request.- * @param idempotencyKey - Optional idempotency key for the request.- * @returns The response from the extract operation.- */- async asyncExtract(- urls: string[],- params?: ExtractParams,- idempotencyKey?: string- ): Promise<ExtractResponse | ErrorResponse> {- const headers = this.prepareHeaders(idempotencyKey);- let jsonData: any = { urls, ...params };- let jsonSchema: any;-- try {- if (!params?.schema) {- jsonSchema = undefined;- } else {- try {- jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);- } catch (_) {- jsonSchema = params.schema;- }- }- } catch (error: any) {- throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);- }-- try {- const response: AxiosResponse = await this.postRequest(- this.apiUrl + `/v1/extract`,- { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },- headers- );-- if (response.status === 200) {- return response.data;- } else {- this.handleError(response, "start extract job");- }- } catch (error: any) {- throw new FirecrawlError(error.message, 500, error.response?.data?.details);- }- return { success: false, error: "Internal server error." };- }-- /**- * Retrieves the status of an extract job.- * @param jobId - The ID of the extract job.- * @returns The status of the extract job.- */- async getExtractStatus(jobId: string): Promise<any> {- try {- const response: AxiosResponse = await this.getRequest(- `${this.apiUrl}/v1/extract/${jobId}`,- this.prepareHeaders()- );-- if (response.status === 200) {- return response.data;- } else {- this.handleError(response, "get extract status");- }- } catch (error: any) {- throw new FirecrawlError(error.message, 500);- }- }-- /**- * Prepares the headers for an API request.- * @param idempotencyKey - Optional key to ensure idempotency.- * @returns The prepared headers.- */- prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {- return {- "Content-Type": "application/json",- Authorization: `Bearer ${this.apiKey}`,- ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),- } as AxiosRequestHeaders & { "x-idempotency-key"?: string };- }-- /**- * Sends a POST request to the specified URL.- * @param url - The URL to send the request to.- * @param data - The data to send in the request.- * @param headers - The headers for the request.- * @returns The response from the POST request.- */- postRequest(- url: string,- data: any,- headers: AxiosRequestHeaders- ): Promise<AxiosResponse> {- return axios.post(url, data, { headers, timeout: (data?.timeout ? (data.timeout + 5000) : undefined) });- }-- /**- * Sends a GET request to the specified URL.- * @param url - The URL to send the request to.- * @param headers - The headers for the request.- * @returns The response from the GET request.- */- async getRequest(- url: string,- headers: AxiosRequestHeaders- ): Promise<AxiosResponse> {- try {- return await axios.get(url, { headers });- } catch (error) {- if (error instanceof AxiosError && error.response) {- return error.response as AxiosResponse;- } else {- throw error;- }- }- }-- /**- * Sends a DELETE request to the specified URL.- * @param url - The URL to send the request to.- * @param headers - The headers for the request.- * @returns The response from the DELETE request.- */- async deleteRequest(- url: string,- headers: AxiosRequestHeaders- ): Promise<AxiosResponse> {- try {- return await axios.delete(url, { headers });- } catch (error) {- if (error instanceof AxiosError && error.response) {- return error.response as AxiosResponse;- } else {- throw error;- }- }- }-- /**- * Monitors the status of a crawl job until completion or failure.- * @param id - The ID of the crawl operation.- * @param headers - The headers for the request.- * @param checkInterval - Interval in seconds for job status checks.- * @param checkUrl - Optional URL to check the status (used for v1 API)- * @returns The final job status or data.- */- async monitorJobStatus(- id: string,- headers: AxiosRequestHeaders,- checkInterval: number- ): Promise<CrawlStatusResponse | ErrorResponse> {- let failedTries = 0;- let networkRetries = 0;- const maxNetworkRetries = 3;-- while (true) {- try {- let statusResponse: AxiosResponse = await this.getRequest(- `${this.apiUrl}/v1/crawl/${id}`,- headers- );-- if (statusResponse.status === 200) {- failedTries = 0;- networkRetries = 0;- let statusData = statusResponse.data;-- if (statusData.status === "completed") {- if ("data" in statusData) {- let data = statusData.data;- while (typeof statusData === 'object' && 'next' in statusData) {- if (data.length === 0) {- break- }- statusResponse = await this.getRequest(statusData.next, headers);- statusData = statusResponse.data;- data = data.concat(statusData.data);- }- statusData.data = data;- return statusData;- } else {- throw new FirecrawlError("Crawl job completed but no data was returned", 500);- }- } else if (- ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)- ) {- checkInterval = Math.max(checkInterval, 2);- await new Promise((resolve) =>- setTimeout(resolve, checkInterval * 1000)- );- } else {- throw new FirecrawlError(- `Crawl job failed or was stopped. Status: ${statusData.status}`,- 500- );- }- } else {- failedTries++;- if (failedTries >= 3) {- this.handleError(statusResponse, "check crawl status");- }- }- } catch (error: any) {- if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {- networkRetries++;- const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);-- await new Promise((resolve) => setTimeout(resolve, backoffDelay));- continue;- }-- throw new FirecrawlError(error, 500);- }- }- }-- /**- * Determines if an error is retryable (transient network error)- * @param error - The error to check- * @returns True if the error should be retried- */- private isRetryableError(error: any): boolean {- if (error instanceof AxiosError) {- if (!error.response) {- const code = error.code;- const message = error.message?.toLowerCase() || '';-- return (- code === 'ECONNRESET' ||- code === 'ETIMEDOUT' ||- code === 'ENOTFOUND' ||- code === 'ECONNREFUSED' ||- message.includes('socket hang up') ||- message.includes('network error') ||- message.includes('timeout')- );- }-- if (error.response?.status === 408 || error.response?.status === 504) {- return true;- }- }-- if (error && typeof error === 'object') {- const code = error.code;- const message = error.message?.toLowerCase() || '';-- if (code === 'ECONNRESET' ||- code === 'ETIMEDOUT' ||- code === 'ENOTFOUND' ||- code === 'ECONNREFUSED' ||- message.includes('socket hang up') ||- message.includes('network error') ||- message.includes('timeout')) {- return true;- }-- if (error.response?.status === 408 || error.response?.status === 504) {- return true;- }- }-- return false;- }-- /**- * Handles errors from API responses.- * @param {AxiosResponse} response - The response from the API.- * @param {string} action - The action being performed when the error occurred.- */- handleError(response: AxiosResponse, action: string): void {- if (!response) {- throw new FirecrawlError(- `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,- 0- );- }-- if ([400, 402, 403, 408, 409, 500].includes(response.status)) {- const errorMessage: string =- response.data.error || "Unknown error occurred";- const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : '';- throw new FirecrawlError(- `Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}${details}`,- response.status,- response?.data?.details- );- } else {- throw new FirecrawlError(- `Unexpected error occurred while trying to ${action}. Status code: ${response.status}`,- response.status- );- }- }-- /**- * Initiates a deep research operation on a given query and polls until completion.- * @param query - The query to research.- * @param params - Parameters for the deep research operation.- * @param onActivity - Optional callback to receive activity updates in real-time.- * @param onSource - Optional callback to receive source updates in real-time.- * @returns The final research results.- */- async deepResearch(- query: string,- params: DeepResearchParams<zt.ZodSchema>,- onActivity?: (activity: {- type: string;- status: string;- message: string;- timestamp: string;- depth: number;- }) => void,- onSource?: (source: {- url: string;- title?: string;- description?: string;- icon?: string;- }) => void- ): Promise<DeepResearchStatusResponse | ErrorResponse> {- try {- const response = await this.asyncDeepResearch(query, params);-- if (!response.success || 'error' in response) {- return { success: false, error: 'error' in response ? response.error : 'Unknown error' };- }-- if (!response.id) {- throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);- }-- const jobId = response.id;- let researchStatus;- let lastActivityCount = 0;- let lastSourceCount = 0;-- while (true) {- researchStatus = await this.checkDeepResearchStatus(jobId);-- if ('error' in researchStatus && !researchStatus.success) {- return researchStatus;- }-- // Stream new activities through the callback if provided- if (onActivity && researchStatus.activities) {- const newActivities = researchStatus.activities.slice(lastActivityCount);- for (const activity of newActivities) {- onActivity(activity);- }- lastActivityCount = researchStatus.activities.length;- }-- // Stream new sources through the callback if provided- if (onSource && researchStatus.sources) {- const newSources = researchStatus.sources.slice(lastSourceCount);- for (const source of newSources) {- onSource(source);- }- lastSourceCount = researchStatus.sources.length;- }-- if (researchStatus.status === "completed") {- return researchStatus;- }-- if (researchStatus.status === "failed") {- throw new FirecrawlError(- `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,- 500- );- }-- if (researchStatus.status !== "processing") {- break;- }-- await new Promise(resolve => setTimeout(resolve, 2000));- }-- return { success: false, error: "Research job terminated unexpectedly" };- } catch (error: any) {- throw new FirecrawlError(error.message, 500, error.response?.data?.details);- }- }-- /**- * Initiates a deep research operation on a given query without polling.- * @param params - Parameters for the deep research operation.- * @returns The response containing the research job ID.- */- async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {- const headers = this.prepareHeaders();- let jsonData: any = { query, ...params, origin: `js-sdk@${this.version}` };-- if (jsonData?.jsonOptions?.schema) {- let schema = jsonData.jsonOptions.schema;- // Try parsing the schema as a Zod schema- try {- schema = zodToJsonSchema(schema);- } catch (error) {- // Ignore error if schema can't be parsed as Zod- }- jsonData = {- ...jsonData,- jsonOptions: {- ...jsonData.jsonOptions,- schema: schema,- },- };- }-- try {- const response: AxiosResponse = await this.postRequest(- `${this.apiUrl}/v1/deep-research`,- jsonData,- headers- );-- if (response.status === 200) {- return response.data;- } else {- this.handleError(response, "start deep research");- }- } catch (error: any) {- if (error.response?.data?.error) {- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);- } else {- throw new FirecrawlError(error.message, 500);- }- }- return { success: false, error: "Internal server error." };- }-- /**- * Checks the status of a deep research operation.- * @param id - The ID of the deep research operation.- * @returns The current status and results of the research operation.- */- async checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {- const headers = this.prepareHeaders();- try {- const response: AxiosResponse = await this.getRequest(- `${this.apiUrl}/v1/deep-research/${id}`,- headers- );-- if (response.status === 200) {- return response.data;- } else if (response.status === 404) {- throw new FirecrawlError("Deep research job not found", 404);- } else {- this.handleError(response, "check deep research status");- }- } catch (error: any) {- if (error.response?.data?.error) {- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);- } else {- throw new FirecrawlError(error.message, 500);- }- }- return { success: false, error: "Internal server error." };- }-- /**- * @deprecated Use deepResearch() instead- * Initiates a deep research operation on a given topic and polls until completion.- * @param topic - The topic to research.- * @param params - Parameters for the deep research operation.- * @param onActivity - Optional callback to receive activity updates in real-time.- * @returns The final research results.- */- async __deepResearch(- topic: string,- params: DeepResearchParams,- onActivity?: (activity: {- type: string;- status: string;- message: string;- timestamp: string;- depth: number;- }) => void- ): Promise<DeepResearchStatusResponse | ErrorResponse> {- try {- const response = await this.__asyncDeepResearch(topic, params);-- if (!response.success || 'error' in response) {- return { success: false, error: 'error' in response ? response.error : 'Unknown error' };- }-- if (!response.id) {- throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);- }-- const jobId = response.id;- let researchStatus;- let lastActivityCount = 0;-- while (true) {- researchStatus = await this.__checkDeepResearchStatus(jobId);-- if ('error' in researchStatus && !researchStatus.success) {- return researchStatus;- }-- // Stream new activities through the callback if provided- if (onActivity && researchStatus.activities) {- const newActivities = researchStatus.activities.slice(lastActivityCount);- for (const activity of newActivities) {- onActivity(activity);- }- lastActivityCount = researchStatus.activities.length;- }-- if (researchStatus.status === "completed") {- return researchStatus;- }-- if (researchStatus.status === "failed") {- throw new FirecrawlError(- `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,- 500- );- }-- if (researchStatus.status !== "processing") {- break;- }-- await new Promise(resolve => setTimeout(resolve, 2000));- }-- return { success: false, error: "Research job terminated unexpectedly" };- } catch (error: any) {- throw new FirecrawlError(error.message, 500, error.response?.data?.details);- }- }-- /**- * @deprecated Use asyncDeepResearch() instead- * Initiates a deep research operation on a given topic without polling.- * @param params - Parameters for the deep research operation.- * @returns The response containing the research job ID.- */- async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {- const headers = this.prepareHeaders();- try {- let jsonData: any = { topic, ...params, origin: `js-sdk@${this.version}` };- const response: AxiosResponse = await this.postRequest(- `${this.apiUrl}/v1/deep-research`,- jsonData,- headers- );-- if (response.status === 200) {- return response.data;- } else {- this.handleError(response, "start deep research");- }- } catch (error: any) {- if (error.response?.data?.error) {- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);- } else {- throw new FirecrawlError(error.message, 500);- }- }- return { success: false, error: "Internal server error." };- }-- /**- * @deprecated Use checkDeepResearchStatus() instead- * Checks the status of a deep research operation.- * @param id - The ID of the deep research operation.- * @returns The current status and results of the research operation.- */- async __checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {- const headers = this.prepareHeaders();- try {- const response: AxiosResponse = await this.getRequest(- `${this.apiUrl}/v1/deep-research/${id}`,- headers- );-- if (response.status === 200) {- return response.data;- } else if (response.status === 404) {- throw new FirecrawlError("Deep research job not found", 404);- } else {- this.handleError(response, "check deep research status");- }- } catch (error: any) {- if (error.response?.data?.error) {- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);- } else {- throw new FirecrawlError(error.message, 500);- }- }- return { success: false, error: "Internal server error." };- }-- /**- * Generates LLMs.txt for a given URL and polls until completion.- * @param url - The URL to generate LLMs.txt from.- * @param params - Parameters for the LLMs.txt generation operation.- * @returns The final generation results.- */- async generateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {- try {- const response = await this.asyncGenerateLLMsText(url, params);-- if (!response.success || 'error' in response) {- return { success: false, error: 'error' in response ? response.error : 'Unknown error' };- }-- if (!response.id) {- throw new FirecrawlError(`Failed to start LLMs.txt generation. No job ID returned.`, 500);- }-- const jobId = response.id;- let generationStatus;-- while (true) {- generationStatus = await this.checkGenerateLLMsTextStatus(jobId);-- if ('error' in generationStatus && !generationStatus.success) {- return generationStatus;- }-- if (generationStatus.status === "completed") {- return generationStatus;- }-- if (generationStatus.status === "failed") {- throw new FirecrawlError(- `LLMs.txt generation job ${generationStatus.status}. Error: ${generationStatus.error}`,- 500- );- }-- if (generationStatus.status !== "processing") {- break;- }-- await new Promise(resolve => setTimeout(resolve, 2000));- }-- return { success: false, error: "LLMs.txt generation job terminated unexpectedly" };- } catch (error: any) {- throw new FirecrawlError(error.message, 500, error.response?.data?.details);- }- }-- /**- * Initiates a LLMs.txt generation operation without polling.- * @param url - The URL to generate LLMs.txt from.- * @param params - Parameters for the LLMs.txt generation operation.- * @returns The response containing the generation job ID.- */- async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {- const headers = this.prepareHeaders();- let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };- try {- const response: AxiosResponse = await this.postRequest(- `${this.apiUrl}/v1/llmstxt`,- jsonData,- headers- );-- if (response.status === 200) {- return response.data;- } else {- this.handleError(response, "start LLMs.txt generation");- }- } catch (error: any) {- if (error.response?.data?.error) {- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);- } else {- throw new FirecrawlError(error.message, 500);- }- }- return { success: false, error: "Internal server error." };- }-- /**- * Checks the status of a LLMs.txt generation operation.- * @param id - The ID of the LLMs.txt generation operation.- * @returns The current status and results of the generation operation.- */- async checkGenerateLLMsTextStatus(id: string): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {- const headers = this.prepareHeaders();- try {- const response: AxiosResponse = await this.getRequest(- `${this.apiUrl}/v1/llmstxt/${id}`,- headers- );-- if (response.status === 200) {- return response.data;- } else if (response.status === 404) {- throw new FirecrawlError("LLMs.txt generation job not found", 404);- } else {- this.handleError(response, "check LLMs.txt generation status");- }- } catch (error: any) {- if (error.response?.data?.error) {- throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);- } else {- throw new FirecrawlError(error.message, 500);- }- }- return { success: false, error: "Internal server error." };- }-}--interface CrawlWatcherEvents {- document: CustomEvent<FirecrawlDocument<undefined>>,- done: CustomEvent<{- status: CrawlStatusResponse["status"];- data: FirecrawlDocument<undefined>[];- }>,- error: CustomEvent<{- status: CrawlStatusResponse["status"],- data: FirecrawlDocument<undefined>[],- error: string,- }>,-}--export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {- private ws: WebSocket;- public data: FirecrawlDocument<undefined>[];- public status: CrawlStatusResponse["status"];- public id: string;-- constructor(id: string, app: FirecrawlApp) {- super();- this.id = id;- // replace `http` with `ws` (`http://` -> `ws://` and `https://` -> `wss://`)- const wsUrl = app.apiUrl.replace(/^http/, "ws");- this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);- this.status = "scraping";- this.data = [];-- type ErrorMessage = {- type: "error",- error: string,- }-- type CatchupMessage = {- type: "catchup",- data: CrawlStatusResponse,- }-- type DocumentMessage = {- type: "document",- data: FirecrawlDocument<undefined>,- }-- type DoneMessage = { type: "done" }-- type Message = ErrorMessage | CatchupMessage | DoneMessage | DocumentMessage;-- const messageHandler = (msg: Message) => {- if (msg.type === "done") {- this.status = "completed";- this.dispatchTypedEvent("done", new CustomEvent("done", {- detail: {- status: this.status,- data: this.data,- id: this.id,- },- }));- } else if (msg.type === "error") {- this.status = "failed";- this.dispatchTypedEvent("error", new CustomEvent("error", {- detail: {- status: this.status,- data: this.data,- error: msg.error,- id: this.id,- },- }));- } else if (msg.type === "catchup") {- this.status = msg.data.status;- this.data.push(...(msg.data.data ?? []));- for (const doc of this.data) {- this.dispatchTypedEvent("document", new CustomEvent("document", {- detail: {- ...doc,- id: this.id,- },- }));- }- } else if (msg.type === "document") {- this.dispatchTypedEvent("document", new CustomEvent("document", {- detail: {- ...msg.data,- id: this.id,- },- }));- }- }-- this.ws.onmessage = ((ev: MessageEvent) => {- if (typeof ev.data !== "string") {- this.ws.close();- return;- }- try {- const msg = JSON.parse(ev.data) as Message;- messageHandler(msg);- } catch (error) {- console.error("Error on message", error);- }- }).bind(this);-- this.ws.onclose = ((ev: CloseEvent) => {- try {- const msg = JSON.parse(ev.reason) as Message;- messageHandler(msg);- } catch (error) {- console.error("Error on close", error);- }- }).bind(this);-- this.ws.onerror = ((_: Event) => {- this.status = "failed"- this.dispatchTypedEvent("error", new CustomEvent("error", {- detail: {- status: this.status,- data: this.data,- error: "WebSocket error",- id: this.id,- },- }));- }).bind(this);- }-- close() {- this.ws.close();- }-}diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsnew file mode 100644index 000000000..a54ae3c32--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -0,0 +1,2146 @@+import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios";+import * as zt from "zod";+import { zodToJsonSchema } from "zod-to-json-schema";+import { TypedEventTarget } from "typescript-event-target";++/**+ * Configuration interface for FirecrawlApp.+ * @param apiKey - Optional API key for authentication.+ * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.+ */+export interface FirecrawlAppConfig {+ apiKey?: string | null;+ apiUrl?: string | null;+}++/**+ * Metadata for a Firecrawl document.+ * Includes various optional properties for document metadata.+ */+export interface FirecrawlDocumentMetadata {+ title?: string;+ description?: string;+ language?: string;+ keywords?: string;+ robots?: string;+ ogTitle?: string;+ ogDescription?: string;+ ogUrl?: string;+ ogImage?: string;+ ogAudio?: string;+ ogDeterminer?: string;+ ogLocale?: string;+ ogLocaleAlternate?: string[];+ ogSiteName?: string;+ ogVideo?: string;+ dctermsCreated?: string;+ dcDateCreated?: string;+ dcDate?: string;+ dctermsType?: string;+ dcType?: string;+ dctermsAudience?: string;+ dctermsSubject?: string;+ dcSubject?: string;+ dcDescription?: string;+ dctermsKeywords?: string;+ modifiedTime?: string;+ publishedTime?: string;+ articleTag?: string;+ articleSection?: string;+ sourceURL?: string;+ statusCode?: number;+ error?: string;+ proxyUsed?: "basic" | "stealth";+ cacheState?: "miss" | "hit";+ cachedAt?: string;+ [key: string]: any; // Allows for additional metadata properties not explicitly defined.+}++/**+ * Document interface for Firecrawl.+ * Represents a document retrieved or processed by Firecrawl.+ */+export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | never) = never> {+ url?: string;+ markdown?: string;+ html?: string;+ rawHtml?: string;+ links?: string[];+ extract?: T;+ json?: T;+ screenshot?: string;+ metadata?: FirecrawlDocumentMetadata;+ actions: ActionsSchema;+ changeTracking?: {+ previousScrapeAt: string | null;+ changeStatus: "new" | "same" | "changed" | "removed";+ visibility: "visible" | "hidden";+ diff?: {+ text: string;+ json: {+ files: Array<{+ from: string | null;+ to: string | null;+ chunks: Array<{+ content: string;+ changes: Array<{+ type: string;+ normal?: boolean;+ ln?: number;+ ln1?: number;+ ln2?: number;+ content: string;+ }>;+ }>;+ }>;+ };+ };+ json?: any;+ };+ // v1 search only+ title?: string;+ description?: string;+}++/**+ * Parameters for scraping operations.+ * Defines the options and configurations available for scraping web content.+ */+export interface CrawlScrapeOptions {+ formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "changeTracking")[];+ headers?: Record<string, string>;+ includeTags?: string[];+ excludeTags?: string[];+ onlyMainContent?: boolean;+ waitFor?: number;+ timeout?: number;+ location?: {+ country?: string;+ languages?: string[];+ };+ mobile?: boolean;+ skipTlsVerification?: boolean;+ removeBase64Images?: boolean;+ blockAds?: boolean;+ proxy?: "basic" | "stealth" | "auto";+ storeInCache?: boolean;+ maxAge?: number;+ parsePDF?: boolean;+}++export type Action = {+ type: "wait",+ milliseconds?: number,+ selector?: string,+} | {+ type: "click",+ selector: string,+ all?: boolean,+} | {+ type: "screenshot",+ fullPage?: boolean,+ quality?: number,+} | {+ type: "write",+ text: string,+} | {+ type: "press",+ key: string,+} | {+ type: "scroll",+ direction?: "up" | "down",+ selector?: string,+} | {+ type: "scrape",+} | {+ type: "executeJavascript",+ script: string,+};++export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema extends (Action[] | undefined) = undefined> extends CrawlScrapeOptions {+ extract?: {+ prompt?: string;+ schema?: LLMSchema;+ systemPrompt?: string;+ };+ jsonOptions?:{+ prompt?: string;+ schema?: LLMSchema;+ systemPrompt?: string;+ }+ changeTrackingOptions?: {+ prompt?: string;+ schema?: any;+ modes?: ("json" | "git-diff")[];+ tag?: string | null;+ }+ actions?: ActionsSchema;+ agent?: AgentOptions;+ zeroDataRetention?: boolean;+}++export interface ActionsResult {+ screenshots: string[];+ scrapes: ({+ url: string;+ html: string;+ })[];+ javascriptReturns: {+ type: string;+ value: unknown+ }[];+}++/**+ * Response interface for scraping operations.+ * Defines the structure of the response received after a scraping operation.+ */+export interface ScrapeResponse<LLMResult = any, ActionsSchema extends (ActionsResult | never) = never> extends FirecrawlDocument<LLMResult, ActionsSchema> {+ success: true;+ warning?: string;+ error?: string;+}++/**+ * Parameters for crawling operations.+ * Includes options for both scraping and mapping during a crawl.+ */+export interface CrawlParams {+ includePaths?: string[];+ excludePaths?: string[];+ maxDepth?: number;+ maxDiscoveryDepth?: number;+ limit?: number;+ allowBackwardLinks?: boolean;+ crawlEntireDomain?: boolean;+ allowExternalLinks?: boolean;+ ignoreSitemap?: boolean;+ scrapeOptions?: CrawlScrapeOptions;+ webhook?: string | {+ url: string;+ headers?: Record<string, string>;+ metadata?: Record<string, string>;+ events?: ["completed", "failed", "page", "started"][number][];+ };+ deduplicateSimilarURLs?: boolean;+ ignoreQueryParameters?: boolean;+ regexOnFullURL?: boolean;+ /**+ * Delay in seconds between scrapes. This helps respect website rate limits.+ * If not provided, the crawler may use the robots.txt crawl delay if available.+ */+ delay?: number;+ allowSubdomains?: boolean;+ maxConcurrency?: number;+ zeroDataRetention?: boolean;+}++/**+ * Response interface for crawling operations.+ * Defines the structure of the response received after initiating a crawl.+ */+export interface CrawlResponse {+ id?: string;+ url?: string;+ success: true;+ error?: string;+}++/**+ * Response interface for batch scrape operations.+ * Defines the structure of the response received after initiating a crawl.+ */+export interface BatchScrapeResponse {+ id?: string;+ url?: string;+ success: true;+ error?: string;+ invalidURLs?: string[];+}++/**+ * Response interface for job status checks.+ * Provides detailed status of a crawl job including progress and results.+ */+export interface CrawlStatusResponse {+ success: true;+ status: "scraping" | "completed" | "failed" | "cancelled";+ completed: number;+ total: number;+ creditsUsed: number;+ expiresAt: Date;+ next?: string;+ data: FirecrawlDocument<undefined>[];+};++/**+ * Response interface for batch scrape job status checks.+ * Provides detailed status of a batch scrape job including progress and results.+ */+export interface BatchScrapeStatusResponse {+ success: true;+ status: "scraping" | "completed" | "failed" | "cancelled";+ completed: number;+ total: number;+ creditsUsed: number;+ expiresAt: Date;+ next?: string;+ data: FirecrawlDocument<undefined>[];+};++/**+ * Parameters for mapping operations.+ * Defines options for mapping URLs during a crawl.+ */+export interface MapParams {+ search?: string;+ ignoreSitemap?: boolean;+ includeSubdomains?: boolean;+ sitemapOnly?: boolean;+ limit?: number;+ timeout?: number;+ useIndex?: boolean;+}++/**+ * Response interface for mapping operations.+ * Defines the structure of the response received after a mapping operation.+ */+export interface MapResponse {+ success: true;+ links?: string[];+ error?: string;+}++/**+ * Parameters for extracting information from URLs.+ * Defines options for extracting information from URLs.+ */+export interface AgentOptions {+ model?: string;+ prompt?: string;+ sessionId?: string;+}++/**+ * Parameters for extracting information from URLs.+ * Defines options for extracting information from URLs.+ */+export interface AgentOptionsExtract {+ model?: string;+ sessionId?: string;+}++export interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {+ prompt?: string;+ schema?: LLMSchema | object;+ systemPrompt?: string;+ allowExternalLinks?: boolean;+ enableWebSearch?: boolean;+ includeSubdomains?: boolean;+ origin?: string;+ showSources?: boolean;+ scrapeOptions?: CrawlScrapeOptions;+ agent?: AgentOptionsExtract;+}++/**+ * Response interface for extracting information from URLs.+ * Defines the structure of the response received after extracting information from URLs.+ */+export interface ExtractResponse<LLMSchema extends zt.ZodSchema = any> {+ success: boolean;+ data: LLMSchema;+ error?: string;+ warning?: string;+ sources?: string[];+}++/**+ * Error response interface.+ * Defines the structure of the response received when an error occurs.+ */+export interface ErrorResponse {+ success: false;+ error: string;+}++/**+ * Custom error class for Firecrawl.+ * Extends the built-in Error class to include a status code.+ */+export class FirecrawlError extends Error {+ statusCode: number;+ details?: any;+ constructor(message: string, statusCode: number, details?: any) {+ super(message);+ this.statusCode = statusCode;+ this.details = details;+ }+}++/**+ * Parameters for search operations.+ * Defines options for searching and scraping search results.+ */+export interface SearchParams {+ limit?: number;+ tbs?: string;+ filter?: string;+ lang?: string;+ country?: string;+ location?: string;+ origin?: string;+ timeout?: number;+ scrapeOptions?: ScrapeParams;+}++/**+ * Response interface for search operations.+ * Defines the structure of the response received after a search operation.+ */+export interface SearchResponse {+ success: boolean;+ data: FirecrawlDocument<undefined>[];+ warning?: string;+ error?: string;+}++/**+ * Response interface for crawl/batch scrape error monitoring.+ */+export interface CrawlErrorsResponse {+ /**+ * Scrapes that errored out + error details+ */+ errors: {+ id: string,+ timestamp?: string,+ url: string,+ error: string,+ }[];++ /**+ * URLs blocked by robots.txt+ */+ robotsBlocked: string[];+};++/**+ * Parameters for deep research operations.+ * Defines options for conducting deep research on a query.+ */+export interface DeepResearchParams<LLMSchema extends zt.ZodSchema = any> {+ /**+ * Maximum depth of research iterations (1-10)+ * @default 7+ */+ maxDepth?: number;+ /**+ * Time limit in seconds (30-300)+ * @default 270+ */+ timeLimit?: number;+ /**+ * Maximum number of URLs to analyze (1-1000)+ * @default 20+ */+ maxUrls?: number;+ /**+ * The prompt to use for the final analysis+ */+ analysisPrompt?: string;+ /**+ * The system prompt to use for the research agent+ */+ systemPrompt?: string;+ /**+ * The formats to use for the final analysis+ */+ formats?: ("markdown" | "json")[];+ /**+ * The JSON options to use for the final analysis+ */+ jsonOptions?:{+ prompt?: string;+ schema?: LLMSchema;+ systemPrompt?: string;+ };+ /**+ * Experimental flag for streaming steps+ */+ // __experimental_streamSteps?: boolean;+}++/**+ * Response interface for deep research operations.+ */+export interface DeepResearchResponse {+ success: boolean;+ id: string;+}++/**+ * Status response interface for deep research operations.+ */+export interface DeepResearchStatusResponse {+ success: boolean;+ data: {+ finalAnalysis: string;+ activities: Array<{+ type: string;+ status: string;+ message: string;+ timestamp: string;+ depth: number;+ }>;+ sources: Array<{+ url: string;+ title: string;+ description: string;+ }>;+ };+ status: "processing" | "completed" | "failed";+ error?: string;+ expiresAt: string;+ currentDepth: number;+ maxDepth: number;+ activities: Array<{+ type: string;+ status: string;+ message: string;+ timestamp: string;+ depth: number;+ }>;+ sources: Array<{+ url: string;+ title: string;+ description: string;+ }>;+ summaries: string[];+}++/**+ * Parameters for LLMs.txt generation operations.+ */+export interface GenerateLLMsTextParams {+ /**+ * Maximum number of URLs to process (1-100)+ * @default 10+ */+ maxUrls?: number;+ /**+ * Whether to show the full LLMs-full.txt in the response+ * @default false+ */+ showFullText?: boolean;+ /**+ * Whether to use cached content if available+ * @default true+ */+ cache?: boolean;+ /**+ * Experimental flag for streaming+ */+ __experimental_stream?: boolean;+}++/**+ * Response interface for LLMs.txt generation operations.+ */+export interface GenerateLLMsTextResponse {+ success: boolean;+ id: string;+}++/**+ * Status response interface for LLMs.txt generation operations.+ */+export interface GenerateLLMsTextStatusResponse {+ success: boolean;+ data: {+ llmstxt: string;+ llmsfulltxt?: string;+ };+ status: "processing" | "completed" | "failed";+ error?: string;+ expiresAt: string;+}++/**+ * Main class for interacting with the Firecrawl API.+ * Provides methods for scraping, searching, crawling, and mapping web content.+ */+export default class FirecrawlApp {+ public apiKey: string;+ public apiUrl: string;+ public version: string = "1.25.1";++ private isCloudService(url: string): boolean {+ return url.includes('api.firecrawl.dev');+ }++ private async getVersion(): Promise<string> {+ try {+ const packageJson = await import('../../package.json', { assert: { type: 'json' } });+ return packageJson.default.version;+ } catch (error) {+ console.error("Error getting version:", error);+ return "1.25.1";+ }+ }++ private async init() {+ this.version = await this.getVersion();+ }++ /**+ * Initializes a new instance of the FirecrawlApp class.+ * @param config - Configuration options for the FirecrawlApp instance.+ */+ constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {+ const baseUrl = apiUrl || "https://api.firecrawl.dev";++ if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {+ throw new FirecrawlError("No API key provided", 401);+ }++ this.apiKey = apiKey || '';+ this.apiUrl = baseUrl;+ this.init();+ }++ /**+ * Scrapes a URL using the Firecrawl API.+ * @param url - The URL to scrape.+ * @param params - Additional parameters for the scrape request.+ * @returns The response from the scrape operation.+ */+ async scrapeUrl<T extends zt.ZodSchema, ActionsSchema extends (Action[] | undefined) = undefined>(+ url: string,+ params?: ScrapeParams<T, ActionsSchema>+ ): Promise<ScrapeResponse<zt.infer<T>, ActionsSchema extends Action[] ? ActionsResult : never> | ErrorResponse> {+ const headers: AxiosRequestHeaders = {+ "Content-Type": "application/json",+ Authorization: `Bearer ${this.apiKey}`,+ } as AxiosRequestHeaders;+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };+ if (jsonData?.extract?.schema) {+ let schema = jsonData.extract.schema;++ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {++ }+ jsonData = {+ ...jsonData,+ extract: {+ ...jsonData.extract,+ schema: schema,+ },+ };+ }++ if (jsonData?.jsonOptions?.schema) {+ let schema = jsonData.jsonOptions.schema;+ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {++ }+ jsonData = {+ ...jsonData,+ jsonOptions: {+ ...jsonData.jsonOptions,+ schema: schema,+ },+ };+ }+ try {+ const response: AxiosResponse = await axios.post(+ this.apiUrl + `/v1/scrape`,+ jsonData,+ { headers, timeout: params?.timeout !== undefined ? (params.timeout + 5000) : undefined },+ );+ if (response.status === 200) {+ const responseData = response.data;+ if (responseData.success) {+ return {+ success: true,+ warning: responseData.warning,+ error: responseData.error,+ ...responseData.data+ };+ } else {+ throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);+ }+ } else {+ this.handleError(response, "scrape URL");+ }+ } catch (error: any) {+ this.handleError(error.response, "scrape URL");+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Searches using the Firecrawl API and optionally scrapes the results.+ * @param query - The search query string.+ * @param params - Optional parameters for the search request.+ * @returns The response from the search operation.+ */+ async search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse> {+ const headers: AxiosRequestHeaders = {+ "Content-Type": "application/json",+ Authorization: `Bearer ${this.apiKey}`,+ } as AxiosRequestHeaders;++ let jsonData: any = {+ query,+ limit: params?.limit ?? 5,+ tbs: params?.tbs,+ filter: params?.filter,+ lang: params?.lang ?? "en",+ country: params?.country ?? "us",+ location: params?.location,+ origin: `js-sdk@${this.version}`,+ timeout: params?.timeout ?? 60000,+ scrapeOptions: params?.scrapeOptions ?? { formats: [] },+ };++ if (jsonData?.scrapeOptions?.extract?.schema) {+ let schema = jsonData.scrapeOptions.extract.schema;++ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {++ }+ jsonData = {+ ...jsonData,+ scrapeOptions: {+ ...jsonData.scrapeOptions,+ extract: {+ ...jsonData.scrapeOptions.extract,+ schema: schema,+ },+ },+ };+ }++ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/search`,+ jsonData,+ headers+ );++ if (response.status === 200) {+ const responseData = response.data;+ if (responseData.success) {+ return {+ success: true,+ data: responseData.data as FirecrawlDocument<any>[],+ warning: responseData.warning,+ };+ } else {+ throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);+ }+ } else {+ this.handleError(response, "search");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error.", data: [] };+ }++ /**+ * Initiates a crawl job for a URL using the Firecrawl API.+ * @param url - The URL to crawl.+ * @param params - Additional parameters for the crawl request.+ * @param pollInterval - Time in seconds for job status checks.+ * @param idempotencyKey - Optional idempotency key for the request.+ * @returns The response from the crawl operation.+ */+ async crawlUrl(+ url: string,+ params?: CrawlParams,+ pollInterval: number = 2,+ idempotencyKey?: string+ ): Promise<CrawlStatusResponse | ErrorResponse> {+ const headers = this.prepareHeaders(idempotencyKey);+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };+ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/crawl`,+ jsonData,+ headers+ );+ if (response.status === 200) {+ const id: string = response.data.id;+ return this.monitorJobStatus(id, headers, pollInterval);+ } else {+ this.handleError(response, "start crawl job");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ async asyncCrawlUrl(+ url: string,+ params?: CrawlParams,+ idempotencyKey?: string+ ): Promise<CrawlResponse | ErrorResponse> {+ const headers = this.prepareHeaders(idempotencyKey);+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };+ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/crawl`,+ jsonData,+ headers+ );+ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start crawl job");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Checks the status of a crawl job using the Firecrawl API.+ * @param id - The ID of the crawl operation.+ * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)+ * @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.+ * @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.+ * @param limit - How many entries to return. Only used when `getAllData = false`.+ * @returns The response containing the job status.+ */+ async checkCrawlStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse> {+ if (!id) {+ throw new FirecrawlError("No crawl ID provided", 400);+ }++ const headers: AxiosRequestHeaders = this.prepareHeaders();+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);+ if (skip !== undefined) {+ targetURL.searchParams.set("skip", skip.toString());+ }+ if (limit !== undefined) {+ targetURL.searchParams.set("limit", limit.toString());+ }++ try {+ const response: AxiosResponse = await this.getRequest(+ targetURL.href,+ headers+ );+ if (response.status === 200) {+ let allData = response.data.data;+ if (getAllData && response.data.status === "completed") {+ let statusData = response.data+ if ("data" in statusData) {+ let data = statusData.data;+ while (typeof statusData === 'object' && 'next' in statusData) {+ if (data.length === 0) {+ break+ }+ statusData = (await this.getRequest(statusData.next, headers)).data;+ data = data.concat(statusData.data);+ }+ allData = data;+ }+ }++ let resp: CrawlStatusResponse | ErrorResponse = {+ success: response.data.success,+ status: response.data.status,+ total: response.data.total,+ completed: response.data.completed,+ creditsUsed: response.data.creditsUsed,+ next: getAllData ? undefined : response.data.next,+ expiresAt: new Date(response.data.expiresAt),+ data: allData+ }++ if (!response.data.success && response.data.error) {+ resp = {+ ...resp,+ success: false,+ error: response.data.error+ } as ErrorResponse;+ }++ if (response.data.next) {+ (resp as CrawlStatusResponse).next = response.data.next;+ }++ return resp;+ } else {+ this.handleError(response, "check crawl status");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Returns information about crawl errors.+ * @param id - The ID of the crawl operation.+ * @returns Information about crawl errors.+ */+ async checkCrawlErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.deleteRequest(+ `${this.apiUrl}/v1/crawl/${id}/errors`,+ headers+ );+ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "check crawl errors");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Cancels a crawl job using the Firecrawl API.+ * @param id - The ID of the crawl operation.+ * @returns The response from the cancel crawl operation.+ */+ async cancelCrawl(id: string): Promise<ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.deleteRequest(+ `${this.apiUrl}/v1/crawl/${id}`,+ headers+ );+ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "cancel crawl job");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Initiates a crawl job and returns a CrawlWatcher to monitor the job via WebSocket.+ * @param url - The URL to crawl.+ * @param params - Additional parameters for the crawl request.+ * @param idempotencyKey - Optional idempotency key for the request.+ * @returns A CrawlWatcher instance to monitor the crawl job.+ */+ async crawlUrlAndWatch(+ url: string,+ params?: CrawlParams,+ idempotencyKey?: string,+ ) {+ const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);++ if (crawl.success && crawl.id) {+ const id = crawl.id;+ return new CrawlWatcher(id, this);+ }++ throw new FirecrawlError("Crawl job failed to start", 400);+ }++ /**+ * Maps a URL using the Firecrawl API.+ * @param url - The URL to map.+ * @param params - Additional parameters for the map request.+ * @returns The response from the map operation.+ */+ async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };++ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/map`,+ jsonData,+ headers+ );+ if (response.status === 200) {+ return response.data as MapResponse;+ } else {+ this.handleError(response, "map");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Initiates a batch scrape job for multiple URLs using the Firecrawl API.+ * @param url - The URLs to scrape.+ * @param params - Additional parameters for the scrape request.+ * @param pollInterval - Time in seconds for job status checks.+ * @param idempotencyKey - Optional idempotency key for the request.+ * @param webhook - Optional webhook for the batch scrape.+ * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.+ * @returns The response from the crawl operation.+ */+ async batchScrapeUrls(+ urls: string[],+ params?: ScrapeParams,+ pollInterval: number = 2,+ idempotencyKey?: string,+ webhook?: CrawlParams["webhook"],+ ignoreInvalidURLs?: boolean,+ maxConcurrency?: number,+ ): Promise<BatchScrapeStatusResponse | ErrorResponse> {+ const headers = this.prepareHeaders(idempotencyKey);+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: `js-sdk@${this.version}` };+ if (jsonData?.extract?.schema) {+ let schema = jsonData.extract.schema;++ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {++ }+ jsonData = {+ ...jsonData,+ extract: {+ ...jsonData.extract,+ schema: schema,+ },+ };+ }+ if (jsonData?.jsonOptions?.schema) {+ let schema = jsonData.jsonOptions.schema;++ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {++ }+ jsonData = {+ ...jsonData,+ jsonOptions: {+ ...jsonData.jsonOptions,+ schema: schema,+ },+ };+ }+ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/batch/scrape`,+ jsonData,+ headers+ );+ if (response.status === 200) {+ const id: string = response.data.id;+ return this.monitorJobStatus(id, headers, pollInterval);+ } else {+ this.handleError(response, "start batch scrape job");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ async asyncBatchScrapeUrls(+ urls: string[],+ params?: ScrapeParams,+ idempotencyKey?: string,+ webhook?: CrawlParams["webhook"],+ ignoreInvalidURLs?: boolean,+ ): Promise<BatchScrapeResponse | ErrorResponse> {+ const headers = this.prepareHeaders(idempotencyKey);+ let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params, origin: `js-sdk@${this.version}` };+ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/batch/scrape`,+ jsonData,+ headers+ );+ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start batch scrape job");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.+ * @param urls - The URL to scrape.+ * @param params - Additional parameters for the scrape request.+ * @param idempotencyKey - Optional idempotency key for the request.+ * @returns A CrawlWatcher instance to monitor the crawl job.+ */+ async batchScrapeUrlsAndWatch(+ urls: string[],+ params?: ScrapeParams,+ idempotencyKey?: string,+ webhook?: CrawlParams["webhook"],+ ignoreInvalidURLs?: boolean,+ ) {+ const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);++ if (crawl.success && crawl.id) {+ const id = crawl.id;+ return new CrawlWatcher(id, this);+ }++ throw new FirecrawlError("Batch scrape job failed to start", 400);+ }++ /**+ * Checks the status of a batch scrape job using the Firecrawl API.+ * @param id - The ID of the batch scrape operation.+ * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)+ * @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.+ * @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.+ * @param limit - How many entries to return. Only used when `getAllData = false`.+ * @returns The response containing the job status.+ */+ async checkBatchScrapeStatus(id?: string, getAllData = false, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse> {+ if (!id) {+ throw new FirecrawlError("No batch scrape ID provided", 400);+ }++ const headers: AxiosRequestHeaders = this.prepareHeaders();+ const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);+ if (skip !== undefined) {+ targetURL.searchParams.set("skip", skip.toString());+ }+ if (limit !== undefined) {+ targetURL.searchParams.set("limit", limit.toString());+ }++ try {+ const response: AxiosResponse = await this.getRequest(+ targetURL.href,+ headers+ );+ if (response.status === 200) {+ let allData = response.data.data;+ if (getAllData && response.data.status === "completed") {+ let statusData = response.data+ if ("data" in statusData) {+ let data = statusData.data;+ while (typeof statusData === 'object' && 'next' in statusData) {+ if (data.length === 0) {+ break+ }+ statusData = (await this.getRequest(statusData.next, headers)).data;+ data = data.concat(statusData.data);+ }+ allData = data;+ }+ }++ let resp: BatchScrapeStatusResponse | ErrorResponse = {+ success: response.data.success,+ status: response.data.status,+ total: response.data.total,+ completed: response.data.completed,+ creditsUsed: response.data.creditsUsed,+ next: getAllData ? undefined : response.data.next,+ expiresAt: new Date(response.data.expiresAt),+ data: allData+ }++ if (!response.data.success && response.data.error) {+ resp = {+ ...resp,+ success: false,+ error: response.data.error+ } as ErrorResponse;+ }++ if (response.data.next) {+ (resp as BatchScrapeStatusResponse).next = response.data.next;+ }++ return resp;+ } else {+ this.handleError(response, "check batch scrape status");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Returns information about batch scrape errors.+ * @param id - The ID of the batch scrape operation.+ * @returns Information about batch scrape errors.+ */+ async checkBatchScrapeErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.deleteRequest(+ `${this.apiUrl}/v1/batch/scrape/${id}/errors`,+ headers+ );+ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "check batch scrape errors");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Extracts information from URLs using the Firecrawl API.+ * Currently in Beta. Expect breaking changes on future minor versions.+ * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.+ * @param params - Additional parameters for the extract request.+ * @returns The response from the extract operation.+ */+ async extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {+ const headers = this.prepareHeaders();++ let jsonData: { urls?: string[] } & ExtractParams<T> = { urls: urls, ...params };+ let jsonSchema: any;+ try {+ if (!params?.schema) {+ jsonSchema = undefined;+ } else {+ try {+ jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);+ } catch (_) {+ jsonSchema = params.schema;+ }+ }+ } catch (error: any) {+ throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);+ }++ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/extract`,+ { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },+ headers+ );++ if (response.status === 200) {+ const jobId = response.data.id;+ let extractStatus;+ do {+ const statusResponse: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/extract/${jobId}`,+ headers+ );+ extractStatus = statusResponse.data;+ if (extractStatus.status === "completed") {+ if (extractStatus.success) {+ return {+ success: true,+ data: extractStatus.data,+ warning: extractStatus.warning,+ error: extractStatus.error,+ sources: extractStatus?.sources || undefined,+ };+ } else {+ throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);+ }+ } else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {+ throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);+ }+ await new Promise(resolve => setTimeout(resolve, 1000)); // Polling interval+ } while (extractStatus.status !== "completed");+ } else {+ this.handleError(response, "extract");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);+ }+ return { success: false, error: "Internal server error."};+ }++ /**+ * Initiates an asynchronous extract job for a URL using the Firecrawl API.+ * @param url - The URL to extract data from.+ * @param params - Additional parameters for the extract request.+ * @param idempotencyKey - Optional idempotency key for the request.+ * @returns The response from the extract operation.+ */+ async asyncExtract(+ urls: string[],+ params?: ExtractParams,+ idempotencyKey?: string+ ): Promise<ExtractResponse | ErrorResponse> {+ const headers = this.prepareHeaders(idempotencyKey);+ let jsonData: any = { urls, ...params };+ let jsonSchema: any;++ try {+ if (!params?.schema) {+ jsonSchema = undefined;+ } else {+ try {+ jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);+ } catch (_) {+ jsonSchema = params.schema;+ }+ }+ } catch (error: any) {+ throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);+ }++ try {+ const response: AxiosResponse = await this.postRequest(+ this.apiUrl + `/v1/extract`,+ { ...jsonData, schema: jsonSchema, origin: `js-sdk@${this.version}` },+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start extract job");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Retrieves the status of an extract job.+ * @param jobId - The ID of the extract job.+ * @returns The status of the extract job.+ */+ async getExtractStatus(jobId: string): Promise<any> {+ try {+ const response: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/extract/${jobId}`,+ this.prepareHeaders()+ );++ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "get extract status");+ }+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500);+ }+ }++ /**+ * Prepares the headers for an API request.+ * @param idempotencyKey - Optional key to ensure idempotency.+ * @returns The prepared headers.+ */+ prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {+ return {+ "Content-Type": "application/json",+ Authorization: `Bearer ${this.apiKey}`,+ ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),+ } as AxiosRequestHeaders & { "x-idempotency-key"?: string };+ }++ /**+ * Sends a POST request to the specified URL.+ * @param url - The URL to send the request to.+ * @param data - The data to send in the request.+ * @param headers - The headers for the request.+ * @returns The response from the POST request.+ */+ postRequest(+ url: string,+ data: any,+ headers: AxiosRequestHeaders+ ): Promise<AxiosResponse> {+ return axios.post(url, data, { headers, timeout: (data?.timeout ? (data.timeout + 5000) : undefined) });+ }++ /**+ * Sends a GET request to the specified URL.+ * @param url - The URL to send the request to.+ * @param headers - The headers for the request.+ * @returns The response from the GET request.+ */+ async getRequest(+ url: string,+ headers: AxiosRequestHeaders+ ): Promise<AxiosResponse> {+ try {+ return await axios.get(url, { headers });+ } catch (error) {+ if (error instanceof AxiosError && error.response) {+ return error.response as AxiosResponse;+ } else {+ throw error;+ }+ }+ }++ /**+ * Sends a DELETE request to the specified URL.+ * @param url - The URL to send the request to.+ * @param headers - The headers for the request.+ * @returns The response from the DELETE request.+ */+ async deleteRequest(+ url: string,+ headers: AxiosRequestHeaders+ ): Promise<AxiosResponse> {+ try {+ return await axios.delete(url, { headers });+ } catch (error) {+ if (error instanceof AxiosError && error.response) {+ return error.response as AxiosResponse;+ } else {+ throw error;+ }+ }+ }++ /**+ * Monitors the status of a crawl job until completion or failure.+ * @param id - The ID of the crawl operation.+ * @param headers - The headers for the request.+ * @param checkInterval - Interval in seconds for job status checks.+ * @param checkUrl - Optional URL to check the status (used for v1 API)+ * @returns The final job status or data.+ */+ async monitorJobStatus(+ id: string,+ headers: AxiosRequestHeaders,+ checkInterval: number+ ): Promise<CrawlStatusResponse | ErrorResponse> {+ let failedTries = 0;+ let networkRetries = 0;+ const maxNetworkRetries = 3;++ while (true) {+ try {+ let statusResponse: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/crawl/${id}`,+ headers+ );++ if (statusResponse.status === 200) {+ failedTries = 0;+ networkRetries = 0;+ let statusData = statusResponse.data;++ if (statusData.status === "completed") {+ if ("data" in statusData) {+ let data = statusData.data;+ while (typeof statusData === 'object' && 'next' in statusData) {+ if (data.length === 0) {+ break+ }+ statusResponse = await this.getRequest(statusData.next, headers);+ statusData = statusResponse.data;+ data = data.concat(statusData.data);+ }+ statusData.data = data;+ return statusData;+ } else {+ throw new FirecrawlError("Crawl job completed but no data was returned", 500);+ }+ } else if (+ ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)+ ) {+ checkInterval = Math.max(checkInterval, 2);+ await new Promise((resolve) =>+ setTimeout(resolve, checkInterval * 1000)+ );+ } else {+ throw new FirecrawlError(+ `Crawl job failed or was stopped. Status: ${statusData.status}`,+ 500+ );+ }+ } else {+ failedTries++;+ if (failedTries >= 3) {+ this.handleError(statusResponse, "check crawl status");+ }+ }+ } catch (error: any) {+ if (this.isRetryableError(error) && networkRetries < maxNetworkRetries) {+ networkRetries++;+ const backoffDelay = Math.min(1000 * Math.pow(2, networkRetries - 1), 10000);++ await new Promise((resolve) => setTimeout(resolve, backoffDelay));+ continue;+ }++ throw new FirecrawlError(error, 500);+ }+ }+ }++ /**+ * Determines if an error is retryable (transient network error)+ * @param error - The error to check+ * @returns True if the error should be retried+ */+ private isRetryableError(error: any): boolean {+ if (error instanceof AxiosError) {+ if (!error.response) {+ const code = error.code;+ const message = error.message?.toLowerCase() || '';++ return (+ code === 'ECONNRESET' ||+ code === 'ETIMEDOUT' ||+ code === 'ENOTFOUND' ||+ code === 'ECONNREFUSED' ||+ message.includes('socket hang up') ||+ message.includes('network error') ||+ message.includes('timeout')+ );+ }++ if (error.response?.status === 408 || error.response?.status === 504) {+ return true;+ }+ }++ if (error && typeof error === 'object') {+ const code = error.code;+ const message = error.message?.toLowerCase() || '';++ if (code === 'ECONNRESET' ||+ code === 'ETIMEDOUT' ||+ code === 'ENOTFOUND' ||+ code === 'ECONNREFUSED' ||+ message.includes('socket hang up') ||+ message.includes('network error') ||+ message.includes('timeout')) {+ return true;+ }++ if (error.response?.status === 408 || error.response?.status === 504) {+ return true;+ }+ }++ return false;+ }++ /**+ * Handles errors from API responses.+ * @param {AxiosResponse} response - The response from the API.+ * @param {string} action - The action being performed when the error occurred.+ */+ handleError(response: AxiosResponse, action: string): void {+ if (!response) {+ throw new FirecrawlError(+ `No response received while trying to ${action}. This may be a network error or the server is unreachable.`,+ 0+ );+ }++ if ([400, 402, 403, 408, 409, 500].includes(response.status)) {+ const errorMessage: string =+ response.data.error || "Unknown error occurred";+ const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : '';+ throw new FirecrawlError(+ `Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}${details}`,+ response.status,+ response?.data?.details+ );+ } else {+ throw new FirecrawlError(+ `Unexpected error occurred while trying to ${action}. Status code: ${response.status}`,+ response.status+ );+ }+ }++ /**+ * Initiates a deep research operation on a given query and polls until completion.+ * @param query - The query to research.+ * @param params - Parameters for the deep research operation.+ * @param onActivity - Optional callback to receive activity updates in real-time.+ * @param onSource - Optional callback to receive source updates in real-time.+ * @returns The final research results.+ */+ async deepResearch(+ query: string,+ params: DeepResearchParams<zt.ZodSchema>,+ onActivity?: (activity: {+ type: string;+ status: string;+ message: string;+ timestamp: string;+ depth: number;+ }) => void,+ onSource?: (source: {+ url: string;+ title?: string;+ description?: string;+ icon?: string;+ }) => void+ ): Promise<DeepResearchStatusResponse | ErrorResponse> {+ try {+ const response = await this.asyncDeepResearch(query, params);++ if (!response.success || 'error' in response) {+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };+ }++ if (!response.id) {+ throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);+ }++ const jobId = response.id;+ let researchStatus;+ let lastActivityCount = 0;+ let lastSourceCount = 0;++ while (true) {+ researchStatus = await this.checkDeepResearchStatus(jobId);++ if ('error' in researchStatus && !researchStatus.success) {+ return researchStatus;+ }++ // Stream new activities through the callback if provided+ if (onActivity && researchStatus.activities) {+ const newActivities = researchStatus.activities.slice(lastActivityCount);+ for (const activity of newActivities) {+ onActivity(activity);+ }+ lastActivityCount = researchStatus.activities.length;+ }++ // Stream new sources through the callback if provided+ if (onSource && researchStatus.sources) {+ const newSources = researchStatus.sources.slice(lastSourceCount);+ for (const source of newSources) {+ onSource(source);+ }+ lastSourceCount = researchStatus.sources.length;+ }++ if (researchStatus.status === "completed") {+ return researchStatus;+ }++ if (researchStatus.status === "failed") {+ throw new FirecrawlError(+ `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,+ 500+ );+ }++ if (researchStatus.status !== "processing") {+ break;+ }++ await new Promise(resolve => setTimeout(resolve, 2000));+ }++ return { success: false, error: "Research job terminated unexpectedly" };+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);+ }+ }++ /**+ * Initiates a deep research operation on a given query without polling.+ * @param params - Parameters for the deep research operation.+ * @returns The response containing the research job ID.+ */+ async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ let jsonData: any = { query, ...params, origin: `js-sdk@${this.version}` };++ if (jsonData?.jsonOptions?.schema) {+ let schema = jsonData.jsonOptions.schema;+ // Try parsing the schema as a Zod schema+ try {+ schema = zodToJsonSchema(schema);+ } catch (error) {+ // Ignore error if schema can't be parsed as Zod+ }+ jsonData = {+ ...jsonData,+ jsonOptions: {+ ...jsonData.jsonOptions,+ schema: schema,+ },+ };+ }++ try {+ const response: AxiosResponse = await this.postRequest(+ `${this.apiUrl}/v1/deep-research`,+ jsonData,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start deep research");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Checks the status of a deep research operation.+ * @param id - The ID of the deep research operation.+ * @returns The current status and results of the research operation.+ */+ async checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/deep-research/${id}`,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else if (response.status === 404) {+ throw new FirecrawlError("Deep research job not found", 404);+ } else {+ this.handleError(response, "check deep research status");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * @deprecated Use deepResearch() instead+ * Initiates a deep research operation on a given topic and polls until completion.+ * @param topic - The topic to research.+ * @param params - Parameters for the deep research operation.+ * @param onActivity - Optional callback to receive activity updates in real-time.+ * @returns The final research results.+ */+ async __deepResearch(+ topic: string,+ params: DeepResearchParams,+ onActivity?: (activity: {+ type: string;+ status: string;+ message: string;+ timestamp: string;+ depth: number;+ }) => void+ ): Promise<DeepResearchStatusResponse | ErrorResponse> {+ try {+ const response = await this.__asyncDeepResearch(topic, params);++ if (!response.success || 'error' in response) {+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };+ }++ if (!response.id) {+ throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);+ }++ const jobId = response.id;+ let researchStatus;+ let lastActivityCount = 0;++ while (true) {+ researchStatus = await this.__checkDeepResearchStatus(jobId);++ if ('error' in researchStatus && !researchStatus.success) {+ return researchStatus;+ }++ // Stream new activities through the callback if provided+ if (onActivity && researchStatus.activities) {+ const newActivities = researchStatus.activities.slice(lastActivityCount);+ for (const activity of newActivities) {+ onActivity(activity);+ }+ lastActivityCount = researchStatus.activities.length;+ }++ if (researchStatus.status === "completed") {+ return researchStatus;+ }++ if (researchStatus.status === "failed") {+ throw new FirecrawlError(+ `Research job ${researchStatus.status}. Error: ${researchStatus.error}`,+ 500+ );+ }++ if (researchStatus.status !== "processing") {+ break;+ }++ await new Promise(resolve => setTimeout(resolve, 2000));+ }++ return { success: false, error: "Research job terminated unexpectedly" };+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);+ }+ }++ /**+ * @deprecated Use asyncDeepResearch() instead+ * Initiates a deep research operation on a given topic without polling.+ * @param params - Parameters for the deep research operation.+ * @returns The response containing the research job ID.+ */+ async __asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ let jsonData: any = { topic, ...params, origin: `js-sdk@${this.version}` };+ const response: AxiosResponse = await this.postRequest(+ `${this.apiUrl}/v1/deep-research`,+ jsonData,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start deep research");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * @deprecated Use checkDeepResearchStatus() instead+ * Checks the status of a deep research operation.+ * @param id - The ID of the deep research operation.+ * @returns The current status and results of the research operation.+ */+ async __checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/deep-research/${id}`,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else if (response.status === 404) {+ throw new FirecrawlError("Deep research job not found", 404);+ } else {+ this.handleError(response, "check deep research status");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Generates LLMs.txt for a given URL and polls until completion.+ * @param url - The URL to generate LLMs.txt from.+ * @param params - Parameters for the LLMs.txt generation operation.+ * @returns The final generation results.+ */+ async generateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {+ try {+ const response = await this.asyncGenerateLLMsText(url, params);++ if (!response.success || 'error' in response) {+ return { success: false, error: 'error' in response ? response.error : 'Unknown error' };+ }++ if (!response.id) {+ throw new FirecrawlError(`Failed to start LLMs.txt generation. No job ID returned.`, 500);+ }++ const jobId = response.id;+ let generationStatus;++ while (true) {+ generationStatus = await this.checkGenerateLLMsTextStatus(jobId);++ if ('error' in generationStatus && !generationStatus.success) {+ return generationStatus;+ }++ if (generationStatus.status === "completed") {+ return generationStatus;+ }++ if (generationStatus.status === "failed") {+ throw new FirecrawlError(+ `LLMs.txt generation job ${generationStatus.status}. Error: ${generationStatus.error}`,+ 500+ );+ }++ if (generationStatus.status !== "processing") {+ break;+ }++ await new Promise(resolve => setTimeout(resolve, 2000));+ }++ return { success: false, error: "LLMs.txt generation job terminated unexpectedly" };+ } catch (error: any) {+ throw new FirecrawlError(error.message, 500, error.response?.data?.details);+ }+ }++ /**+ * Initiates a LLMs.txt generation operation without polling.+ * @param url - The URL to generate LLMs.txt from.+ * @param params - Parameters for the LLMs.txt generation operation.+ * @returns The response containing the generation job ID.+ */+ async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ let jsonData: any = { url, ...params, origin: `js-sdk@${this.version}` };+ try {+ const response: AxiosResponse = await this.postRequest(+ `${this.apiUrl}/v1/llmstxt`,+ jsonData,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "start LLMs.txt generation");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }++ /**+ * Checks the status of a LLMs.txt generation operation.+ * @param id - The ID of the LLMs.txt generation operation.+ * @returns The current status and results of the generation operation.+ */+ async checkGenerateLLMsTextStatus(id: string): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/llmstxt/${id}`,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else if (response.status === 404) {+ throw new FirecrawlError("LLMs.txt generation job not found", 404);+ } else {+ this.handleError(response, "check LLMs.txt generation status");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }+}++interface CrawlWatcherEvents {+ document: CustomEvent<FirecrawlDocument<undefined>>,+ done: CustomEvent<{+ status: CrawlStatusResponse["status"];+ data: FirecrawlDocument<undefined>[];+ }>,+ error: CustomEvent<{+ status: CrawlStatusResponse["status"],+ data: FirecrawlDocument<undefined>[],+ error: string,+ }>,+}++export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {+ private ws: WebSocket;+ public data: FirecrawlDocument<undefined>[];+ public status: CrawlStatusResponse["status"];+ public id: string;++ constructor(id: string, app: FirecrawlApp) {+ super();+ this.id = id;+ // replace `http` with `ws` (`http://` -> `ws://` and `https://` -> `wss://`)+ const wsUrl = app.apiUrl.replace(/^http/, "ws");+ this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);+ this.status = "scraping";+ this.data = [];++ type ErrorMessage = {+ type: "error",+ error: string,+ }++ type CatchupMessage = {+ type: "catchup",+ data: CrawlStatusResponse,+ }++ type DocumentMessage = {+ type: "document",+ data: FirecrawlDocument<undefined>,+ }++ type DoneMessage = { type: "done" }++ type Message = ErrorMessage | CatchupMessage | DoneMessage | DocumentMessage;++ const messageHandler = (msg: Message) => {+ if (msg.type === "done") {+ this.status = "completed";+ this.dispatchTypedEvent("done", new CustomEvent("done", {+ detail: {+ status: this.status,+ data: this.data,+ id: this.id,+ },+ }));+ } else if (msg.type === "error") {+ this.status = "failed";+ this.dispatchTypedEvent("error", new CustomEvent("error", {+ detail: {+ status: this.status,+ data: this.data,+ error: msg.error,+ id: this.id,+ },+ }));+ } else if (msg.type === "catchup") {+ this.status = msg.data.status;+ this.data.push(...(msg.data.data ?? []));+ for (const doc of this.data) {+ this.dispatchTypedEvent("document", new CustomEvent("document", {+ detail: {+ ...doc,+ id: this.id,+ },+ }));+ }+ } else if (msg.type === "document") {+ this.dispatchTypedEvent("document", new CustomEvent("document", {+ detail: {+ ...msg.data,+ id: this.id,+ },+ }));+ }+ }++ this.ws.onmessage = ((ev: MessageEvent) => {+ if (typeof ev.data !== "string") {+ this.ws.close();+ return;+ }+ try {+ const msg = JSON.parse(ev.data) as Message;+ messageHandler(msg);+ } catch (error) {+ console.error("Error on message", error);+ }+ }).bind(this);++ this.ws.onclose = ((ev: CloseEvent) => {+ try {+ const msg = JSON.parse(ev.reason) as Message;+ messageHandler(msg);+ } catch (error) {+ console.error("Error on close", error);+ }+ }).bind(this);++ this.ws.onerror = ((_: Event) => {+ this.status = "failed"+ this.dispatchTypedEvent("error", new CustomEvent("error", {+ detail: {+ status: this.status,+ data: this.data,+ error: "WebSocket error",+ id: this.id,+ },+ }));+ }).bind(this);+ }++ close() {+ this.ws.close();+ }+}diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsnew file mode 100644index 000000000..8987f6de5--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -0,0 +1,151 @@+import { HttpClient } from "./utils/httpClient";+import { scrape } from "./methods/scrape";+import { search } from "./methods/search";+import { map as mapMethod } from "./methods/map";+import {+ startCrawl,+ getCrawlStatus,+ cancelCrawl,+ crawl as crawlWaiter,+ getCrawlErrors,+ getActiveCrawls,+ crawlParamsPreview,+} from "./methods/crawl";+import {+ startBatchScrape,+ getBatchScrapeStatus,+ getBatchScrapeErrors,+ cancelBatchScrape,+ batchScrape as batchWaiter,+} from "./methods/batch";+import { startExtract, getExtractStatus, extract as extractWaiter } from "./methods/extract";+import { getConcurrency, getCreditUsage, getTokenUsage } from "./methods/usage";+import type {+ Document,+ ScrapeOptions,+ SearchData,+ SearchRequest,+ MapData,+ MapOptions,+ CrawlResponse,+ CrawlJob,+ CrawlErrorsResponse,+ ActiveCrawlsResponse,+ BatchScrapeResponse,+ BatchScrapeJob,+ ExtractResponse,+} from "./types";+import { Watcher } from "./watcher";+import type { WatcherOptions } from "./watcher";++export interface FirecrawlClientOptions {+ apiKey?: string | null;+ apiUrl?: string | null;+ timeoutMs?: number;+ maxRetries?: number;+ backoffFactor?: number;+}++export class FirecrawlClient {+ private readonly http: HttpClient;++ constructor(options: FirecrawlClientOptions = {}) {+ const apiKey = options.apiKey ?? process.env.FIRECRAWL_API_KEY ?? "";+ const apiUrl = (options.apiUrl ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev").replace(/\/$/, "");+ if (!apiKey) {+ throw new Error("API key is required. Set FIRECRAWL_API_KEY env or pass apiKey.");+ }+ this.http = new HttpClient({+ apiKey,+ apiUrl,+ timeoutMs: options.timeoutMs,+ maxRetries: options.maxRetries,+ backoffFactor: options.backoffFactor,+ });+ }++ // Scrape+ async scrape(url: string, options?: ScrapeOptions): Promise<Document> {+ return scrape(this.http, url, options);+ }++ // Search+ async search(query: string, req: Omit<SearchRequest, "query"> = {}): Promise<SearchData> {+ return search(this.http, { query, ...req });+ }++ // Map+ async map(url: string, options?: MapOptions): Promise<MapData> {+ return mapMethod(this.http, url, options);+ }++ // Crawl+ async startCrawl(url: string, req: Omit<Parameters<typeof startCrawl>[1], "url"> = {}): Promise<CrawlResponse> {+ return startCrawl(this.http, { url, ...(req as any) });+ }+ async getCrawlStatus(jobId: string): Promise<CrawlJob> {+ return getCrawlStatus(this.http, jobId);+ }+ async cancelCrawl(jobId: string): Promise<boolean> {+ return cancelCrawl(this.http, jobId);+ }+ async crawl(url: string, req: Omit<Parameters<typeof startCrawl>[1], "url"> & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {+ return crawlWaiter(this.http, { url, ...(req as any) }, req.pollInterval, req.timeout);+ }+ async getCrawlErrors(crawlId: string): Promise<CrawlErrorsResponse> {+ return getCrawlErrors(this.http, crawlId);+ }+ async getActiveCrawls(): Promise<ActiveCrawlsResponse> {+ return getActiveCrawls(this.http);+ }+ async crawlParamsPreview(url: string, prompt: string): Promise<Record<string, unknown>> {+ return crawlParamsPreview(this.http, url, prompt);+ }++ // Batch+ async startBatchScrape(urls: string[], opts?: Parameters<typeof startBatchScrape>[2]): Promise<BatchScrapeResponse> {+ return startBatchScrape(this.http, urls, opts);+ }+ async getBatchScrapeStatus(jobId: string): Promise<BatchScrapeJob> {+ return getBatchScrapeStatus(this.http, jobId);+ }+ async getBatchScrapeErrors(jobId: string): Promise<CrawlErrorsResponse> {+ return getBatchScrapeErrors(this.http, jobId);+ }+ async cancelBatchScrape(jobId: string): Promise<boolean> {+ return cancelBatchScrape(this.http, jobId);+ }+ async batchScrape(urls: string[], opts?: Parameters<typeof startBatchScrape>[2] & { pollInterval?: number; timeout?: number }): Promise<BatchScrapeJob> {+ return batchWaiter(this.http, urls, opts);+ }++ // Extract+ async startExtract(args: Parameters<typeof startExtract>[1]): Promise<ExtractResponse> {+ return startExtract(this.http, args);+ }+ async getExtractStatus(jobId: string): Promise<ExtractResponse> {+ return getExtractStatus(this.http, jobId);+ }+ async extract(args: Parameters<typeof startExtract>[1] & { pollInterval?: number; timeout?: number }): Promise<ExtractResponse> {+ return extractWaiter(this.http, args);+ }++ // Usage+ async getConcurrency() {+ return getConcurrency(this.http);+ }+ async getCreditUsage() {+ return getCreditUsage(this.http);+ }+ async getTokenUsage() {+ return getTokenUsage(this.http);+ }++ // Watcher+ watcher(jobId: string, opts: WatcherOptions = {}): Watcher {+ return new Watcher(this.http, jobId, opts);+ }+}++export default FirecrawlClient;+diff --git a/apps/js-sdk/firecrawl/src/v2/methods/batch.ts b/apps/js-sdk/firecrawl/src/v2/methods/batch.tsnew file mode 100644index 000000000..796dd54c4--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/methods/batch.ts@@ -0,0 +1,131 @@+import {+ type BatchScrapeJob,+ type BatchScrapeResponse,+ type CrawlErrorsResponse,+ type Document,+ type ScrapeOptions,+ type WebhookConfig,+} from "../types";+import { HttpClient } from "../utils/httpClient";+import { ensureValidScrapeOptions } from "../utils/validation";+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";++export interface StartBatchOptions {+ options?: ScrapeOptions;+ webhook?: string | WebhookConfig;+ appendToId?: string;+ ignoreInvalidURLs?: boolean;+ maxConcurrency?: number;+ zeroDataRetention?: boolean;+ integration?: string;+ idempotencyKey?: string;+}++export async function startBatchScrape(+ http: HttpClient,+ urls: string[],+ {+ options,+ webhook,+ appendToId,+ ignoreInvalidURLs,+ maxConcurrency,+ zeroDataRetention,+ integration,+ idempotencyKey,+ }: StartBatchOptions = {}+): Promise<BatchScrapeResponse> {+ if (!Array.isArray(urls) || urls.length === 0) throw new Error("URLs list cannot be empty");+ const payload: Record<string, unknown> = { urls };+ if (options) {+ ensureValidScrapeOptions(options);+ Object.assign(payload, options);+ }+ if (webhook != null) payload.webhook = webhook;+ if (appendToId != null) payload.appendToId = appendToId;+ if (ignoreInvalidURLs != null) payload.ignoreInvalidURLs = ignoreInvalidURLs;+ if (maxConcurrency != null) payload.maxConcurrency = maxConcurrency;+ if (zeroDataRetention != null) payload.zeroDataRetention = zeroDataRetention;+ if (integration != null) payload.integration = integration;++ try {+ const headers = http.prepareHeaders(idempotencyKey);+ const res = await http.post<{ success: boolean; id: string; url: string; invalidURLs?: string[]; error?: string }>("/v2/batch/scrape", payload, headers);+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "start batch scrape");+ return { id: res.data.id, url: res.data.url, invalidURLs: res.data.invalidURLs || undefined };+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "start batch scrape");+ throw err;+ }+}++export async function getBatchScrapeStatus(http: HttpClient, jobId: string): Promise<BatchScrapeJob> {+ try {+ const res = await http.get<{ success: boolean; status: BatchScrapeJob["status"]; completed?: number; total?: number; creditsUsed?: number; expiresAt?: string; next?: string | null; data?: Document[] }>(`/v2/batch/scrape/${jobId}`);+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get batch scrape status");+ const body = res.data;+ return {+ status: body.status,+ completed: body.completed ?? 0,+ total: body.total ?? 0,+ creditsUsed: body.creditsUsed,+ expiresAt: body.expiresAt,+ next: body.next ?? null,+ data: (body.data || []) as Document[],+ };+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get batch scrape status");+ throw err;+ }+}++export async function cancelBatchScrape(http: HttpClient, jobId: string): Promise<boolean> {+ try {+ const res = await http.delete<{ status: string }>(`/v2/batch/scrape/${jobId}`);+ if (res.status !== 200) throwForBadResponse(res, "cancel batch scrape");+ return res.data?.status === "cancelled";+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "cancel batch scrape");+ throw err;+ }+}++export async function getBatchScrapeErrors(http: HttpClient, jobId: string): Promise<CrawlErrorsResponse> {+ try {+ const res = await http.get<{ success?: boolean; data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] } }>(`/v2/batch/scrape/${jobId}/errors`);+ if (res.status !== 200) throwForBadResponse(res, "get batch scrape errors");+ const payload = res.data?.data ?? (res.data as any);+ return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get batch scrape errors");+ throw err;+ }+}++export async function waitForBatchCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<BatchScrapeJob> {+ const start = Date.now();+ while (true) {+ const status = await getBatchScrapeStatus(http, jobId);+ if (["completed", "failed", "cancelled"].includes(status.status)) return status;+ if (timeout != null && Date.now() - start > timeout * 1000) {+ throw new Error(`Batch scrape job ${jobId} did not complete within ${timeout} seconds`);+ }+ await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));+ }+}++export async function batchScrape(+ http: HttpClient,+ urls: string[],+ opts: StartBatchOptions & { pollInterval?: number; timeout?: number } = {}+): Promise<BatchScrapeJob> {+ const start = await startBatchScrape(http, urls, opts);+ return waitForBatchCompletion(http, start.id, opts.pollInterval ?? 2, opts.timeout);+}++export function chunkUrls(urls: string[], chunkSize = 100): string[][] {+ const chunks: string[][] = [];+ for (let i = 0; i < urls.length; i += chunkSize) chunks.push(urls.slice(i, i + chunkSize));+ return chunks;+}+diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.tsnew file mode 100644index 000000000..74a6bfd4c--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts@@ -0,0 +1,160 @@+import {+ type ActiveCrawlsResponse,+ type CrawlErrorsResponse,+ type CrawlJob,+ type CrawlResponse,+ type Document,+ type ScrapeOptions,+ type WebhookConfig,+} from "../types";+import { HttpClient } from "../utils/httpClient";+import { ensureValidScrapeOptions } from "../utils/validation";+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";++export interface CrawlRequest {+ url: string;+ prompt?: string | null;+ excludePaths?: string[] | null;+ includePaths?: string[] | null;+ maxDiscoveryDepth?: number | null;+ ignoreSitemap?: boolean;+ ignoreQueryParameters?: boolean;+ limit?: number | null;+ crawlEntireDomain?: boolean;+ allowExternalLinks?: boolean;+ allowSubdomains?: boolean;+ delay?: number | null;+ maxConcurrency?: number | null;+ webhook?: string | WebhookConfig | null;+ scrapeOptions?: ScrapeOptions | null;+ zeroDataRetention?: boolean;+}++function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {+ if (!request.url || !request.url.trim()) throw new Error("URL cannot be empty");+ const data: Record<string, unknown> = { url: request.url.trim() };+ if (request.prompt) data.prompt = request.prompt;+ if (request.excludePaths) data.excludePaths = request.excludePaths;+ if (request.includePaths) data.includePaths = request.includePaths;+ if (request.maxDiscoveryDepth != null) data.maxDiscoveryDepth = request.maxDiscoveryDepth;+ if (request.ignoreSitemap != null) data.ignoreSitemap = request.ignoreSitemap;+ if (request.ignoreQueryParameters != null) data.ignoreQueryParameters = request.ignoreQueryParameters;+ if (request.limit != null) data.limit = request.limit;+ if (request.crawlEntireDomain != null) data.crawlEntireDomain = request.crawlEntireDomain;+ if (request.allowExternalLinks != null) data.allowExternalLinks = request.allowExternalLinks;+ if (request.allowSubdomains != null) data.allowSubdomains = request.allowSubdomains;+ if (request.delay != null) data.delay = request.delay;+ if (request.maxConcurrency != null) data.maxConcurrency = request.maxConcurrency;+ if (request.webhook != null) data.webhook = request.webhook;+ if (request.scrapeOptions) {+ ensureValidScrapeOptions(request.scrapeOptions);+ data.scrapeOptions = request.scrapeOptions;+ }+ if (request.zeroDataRetention != null) data.zeroDataRetention = request.zeroDataRetention;+ return data;+}++export async function startCrawl(http: HttpClient, request: CrawlRequest): Promise<CrawlResponse> {+ const payload = prepareCrawlPayload(request);+ try {+ const res = await http.post<{ success: boolean; id: string; url: string; error?: string }>("/v2/crawl", payload);+ if (res.status !== 200 || !res.data?.success) {+ throwForBadResponse(res, "start crawl");+ }+ return { id: res.data.id, url: res.data.url };+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "start crawl");+ throw err;+ }+}++export async function getCrawlStatus(http: HttpClient, jobId: string): Promise<CrawlJob> {+ try {+ const res = await http.get<{ success: boolean; status: CrawlJob["status"]; completed?: number; total?: number; creditsUsed?: number; expiresAt?: string; next?: string | null; data?: Document[] }>(`/v2/crawl/${jobId}`);+ if (res.status !== 200 || !res.data?.success) {+ throwForBadResponse(res, "get crawl status");+ }+ const body = res.data;+ return {+ status: body.status,+ completed: body.completed ?? 0,+ total: body.total ?? 0,+ creditsUsed: body.creditsUsed,+ expiresAt: body.expiresAt,+ next: body.next ?? null,+ data: (body.data || []) as Document[],+ };+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get crawl status");+ throw err;+ }+}++export async function cancelCrawl(http: HttpClient, jobId: string): Promise<boolean> {+ try {+ const res = await http.delete<{ status: string }>(`/v2/crawl/${jobId}`);+ if (res.status !== 200) throwForBadResponse(res, "cancel crawl");+ return res.data?.status === "cancelled";+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "cancel crawl");+ throw err;+ }+}++export async function waitForCrawlCompletion(http: HttpClient, jobId: string, pollInterval = 2, timeout?: number): Promise<CrawlJob> {+ const start = Date.now();+ while (true) {+ const status = await getCrawlStatus(http, jobId);+ if (["completed", "failed", "cancelled"].includes(status.status)) return status;+ if (timeout != null && Date.now() - start > timeout * 1000) {+ throw new Error(`Crawl job ${jobId} did not complete within ${timeout} seconds`);+ }+ await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));+ }+}++export async function crawl(http: HttpClient, request: CrawlRequest, pollInterval = 2, timeout?: number): Promise<CrawlJob> {+ const started = await startCrawl(http, request);+ return waitForCrawlCompletion(http, started.id, pollInterval, timeout);+}++export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {+ try {+ const res = await http.get<{ success?: boolean; data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] } }>(`/v2/crawl/${crawlId}/errors`);+ if (res.status !== 200) throwForBadResponse(res, "get crawl errors");+ const payload = res.data?.data ?? (res.data as any);+ return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get crawl errors");+ throw err;+ }+}++export async function getActiveCrawls(http: HttpClient): Promise<ActiveCrawlsResponse> {+ try {+ const res = await http.get<{ success: boolean; data?: { crawls: Array<{ id: string; teamId?: string; team_id?: string; url: string; options?: any }> } }>(`/v2/crawl/active`);+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get active crawls");+ const crawlsIn = res.data?.data?.crawls || [];+ const crawls = crawlsIn.map((c) => ({ id: c.id, teamId: (c as any).teamId ?? (c as any).team_id, url: c.url, options: c.options ?? null }));+ return { success: true, crawls };+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get active crawls");+ throw err;+ }+}++export async function crawlParamsPreview(http: HttpClient, url: string, prompt: string): Promise<Record<string, unknown>> {+ if (!url || !url.trim()) throw new Error("URL cannot be empty");+ if (!prompt || !prompt.trim()) throw new Error("Prompt cannot be empty");+ try {+ const res = await http.post<{ success: boolean; data?: Record<string, unknown>; warning?: string }>("/v2/crawl/params-preview", { url: url.trim(), prompt });+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "crawl params preview");+ const data = res.data.data || {};+ if (res.data.warning) (data as any).warning = res.data.warning;+ return data;+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "crawl params preview");+ throw err;+ }+}+diff --git a/apps/js-sdk/firecrawl/src/v2/methods/extract.ts b/apps/js-sdk/firecrawl/src/v2/methods/extract.tsnew file mode 100644index 000000000..ebcd2f52d--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/methods/extract.ts@@ -0,0 +1,80 @@+import { type ExtractResponse, type ScrapeOptions } from "../types";+import { HttpClient } from "../utils/httpClient";+import { ensureValidScrapeOptions } from "../utils/validation";+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";++function prepareExtractPayload(args: {+ urls?: string[];+ prompt?: string;+ schema?: Record<string, unknown>;+ systemPrompt?: string;+ allowExternalLinks?: boolean;+ enableWebSearch?: boolean;+ showSources?: boolean;+ scrapeOptions?: ScrapeOptions;+ ignoreInvalidURLs?: boolean;+}): Record<string, unknown> {+ const body: Record<string, unknown> = {};+ if (args.urls) body.urls = args.urls;+ if (args.prompt != null) body.prompt = args.prompt;+ if (args.schema != null) body.schema = args.schema;+ if (args.systemPrompt != null) body.systemPrompt = args.systemPrompt;+ if (args.allowExternalLinks != null) body.allowExternalLinks = args.allowExternalLinks;+ if (args.enableWebSearch != null) body.enableWebSearch = args.enableWebSearch;+ if (args.showSources != null) body.showSources = args.showSources;+ if (args.ignoreInvalidURLs != null) body.ignoreInvalidURLs = args.ignoreInvalidURLs;+ if (args.scrapeOptions) {+ ensureValidScrapeOptions(args.scrapeOptions);+ body.scrapeOptions = args.scrapeOptions;+ }+ return body;+}++export async function startExtract(http: HttpClient, args: Parameters<typeof prepareExtractPayload>[0]): Promise<ExtractResponse> {+ const payload = prepareExtractPayload(args);+ try {+ const res = await http.post<ExtractResponse>("/v2/extract", payload);+ if (res.status !== 200) throwForBadResponse(res, "extract");+ return res.data;+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "extract");+ throw err;+ }+}++export async function getExtractStatus(http: HttpClient, jobId: string): Promise<ExtractResponse> {+ try {+ const res = await http.get<ExtractResponse>(`/v2/extract/${jobId}`);+ if (res.status !== 200) throwForBadResponse(res, "extract status");+ return res.data;+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "extract status");+ throw err;+ }+}++export async function waitExtract(+ http: HttpClient,+ jobId: string,+ pollInterval = 2,+ timeout?: number+): Promise<ExtractResponse> {+ const start = Date.now();+ while (true) {+ const status = await getExtractStatus(http, jobId);+ if (["completed", "failed", "cancelled"].includes(status.status || "")) return status;+ if (timeout != null && Date.now() - start > timeout * 1000) return status;+ await new Promise((r) => setTimeout(r, Math.max(1000, pollInterval * 1000)));+ }+}++export async function extract(+ http: HttpClient,+ args: Parameters<typeof prepareExtractPayload>[0] & { pollInterval?: number; timeout?: number }+): Promise<ExtractResponse> {+ const started = await startExtract(http, args);+ const jobId = started.id;+ if (!jobId) return started;+ return waitExtract(http, jobId, args.pollInterval ?? 2, args.timeout);+}+diff --git a/apps/js-sdk/firecrawl/src/v2/methods/map.ts b/apps/js-sdk/firecrawl/src/v2/methods/map.tsnew file mode 100644index 000000000..8b41f772e--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/methods/map.ts@@ -0,0 +1,37 @@+import { type MapData, type MapOptions, type SearchResult } from "../types";+import { HttpClient } from "../utils/httpClient";+import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";++function prepareMapPayload(url: string, options?: MapOptions): Record<string, unknown> {+ if (!url || !url.trim()) throw new Error("URL cannot be empty");+ const payload: Record<string, unknown> = { url: url.trim() };+ if (options) {+ if (options.sitemap != null) payload.sitemap = options.sitemap;+ if (options.search != null) payload.search = options.search;+ if (options.includeSubdomains != null) payload.includeSubdomains = options.includeSubdomains;+ if (options.limit != null) payload.limit = options.limit;+ if (options.timeout != null) payload.timeout = options.timeout;+ }+ return payload;+}++export async function map(http: HttpClient, url: string, options?: MapOptions): Promise<MapData> {+ const payload = prepareMapPayload(url, options);+ try {+ const res = await http.post<{ success: boolean; error?: string; links?: Array<string | SearchResult> }>("/v2/map", payload);+ if (res.status !== 200 || !res.data?.success) {+ throwForBadResponse(res, "map");+ }+ const linksIn = res.data.links || [];+ const links: SearchResult[] = [];+ for (const item of linksIn) {+ if (typeof item === "string") links.push({ url: item });+ else if (item && typeof item === "object") links.push({ url: item.url, title: (item as any).title, description: (item as any).description });+ }+ return { links };+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "map");+ throw err;+ }+}+diff --git a/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts b/apps/js-sdk/firecrawl/src/v2/methods/scrape.tsnew file mode 100644index 000000000..c2ddfc045--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/methods/scrape.ts@@ -0,0 +1,26 @@+import { type Document, type ScrapeOptions } from "../types";+import { HttpClient } from "../utils/httpClient";+import { ensureValidScrapeOptions } from "../utils/validation";+import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";++export async function scrape(http: HttpClient, url: string, options?: ScrapeOptions): Promise<Document> {+ if (!url || !url.trim()) {+ throw new Error("URL cannot be empty");+ }+ if (options) ensureValidScrapeOptions(options);++ const payload: Record<string, unknown> = { url: url.trim() };+ if (options) Object.assign(payload, options);++ try {+ const res = await http.post<{ success: boolean; data?: Document; error?: string }>("/v2/scrape", payload);+ if (res.status !== 200 || !res.data?.success) {+ throwForBadResponse(res, "scrape");+ }+ return (res.data.data || {}) as Document;+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "scrape");+ throw err;+ }+}+diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.tsnew file mode 100644index 000000000..65e793140--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts@@ -0,0 +1,60 @@+import { type Document, type SearchData, type SearchRequest, type SearchResult, type ScrapeOptions } from "../types";+import { HttpClient } from "../utils/httpClient";+import { ensureValidScrapeOptions } from "../utils/validation";+import { throwForBadResponse, normalizeAxiosError } from "../utils/errorHandler";++function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {+ if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");+ if (req.limit != null && req.limit <= 0) throw new Error("limit must be positive");+ if (req.timeout != null && req.timeout <= 0) throw new Error("timeout must be positive");+ const payload: Record<string, unknown> = {+ query: req.query,+ };+ if (req.sources) payload.sources = req.sources;+ if (req.limit != null) payload.limit = req.limit;+ if (req.tbs != null) payload.tbs = req.tbs;+ if (req.location != null) payload.location = req.location;+ if (req.ignoreInvalidURLs != null) payload.ignoreInvalidURLs = req.ignoreInvalidURLs;+ if (req.timeout != null) payload.timeout = req.timeout;+ if (req.scrapeOptions) {+ ensureValidScrapeOptions(req.scrapeOptions as ScrapeOptions);+ payload.scrapeOptions = req.scrapeOptions;+ }+ return payload;+}++export async function search(http: HttpClient, request: SearchRequest): Promise<SearchData> {+ const payload = prepareSearchPayload(request);+ try {+ const res = await http.post<{ success: boolean; data?: Record<string, unknown>; error?: string }>("/v2/search", payload);+ if (res.status !== 200 || !res.data?.success) {+ throwForBadResponse(res, "search");+ }+ const data = (res.data.data || {}) as Record<string, any>;+ const out: SearchData = {};+ for (const key of Object.keys(data)) {+ const arr = data[key];+ if (Array.isArray(arr)) {+ const results: Array<SearchResult | Document> = [] as any;+ for (const item of arr) {+ if (item && typeof item === "object") {+ // If scraped page fields present, treat as Document; otherwise SearchResult+ if ("markdown" in item || "html" in item || "rawHtml" in item || "links" in item || "screenshot" in item || "changeTracking" in item) {+ results.push(item as Document);+ } else {+ results.push({ url: item.url, title: item.title, description: item.description } as SearchResult);+ }+ } else if (typeof item === "string") {+ results.push({ url: item } as SearchResult);+ }+ }+ (out as any)[key] = results;+ }+ }+ return out;+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "search");+ throw err;+ }+}+diff --git a/apps/js-sdk/firecrawl/src/v2/methods/usage.ts b/apps/js-sdk/firecrawl/src/v2/methods/usage.tsnew file mode 100644index 000000000..4138a70ec--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/methods/usage.ts@@ -0,0 +1,39 @@+import type { ConcurrencyCheck, CreditUsage, TokenUsage } from "../types";+import { HttpClient } from "../utils/httpClient";+import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";++export async function getConcurrency(http: HttpClient): Promise<ConcurrencyCheck> {+ try {+ const res = await http.get<{ success: boolean; data?: { concurrency: number; maxConcurrency: number } }>("/v2/concurrency-check");+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get concurrency");+ const d = res.data.data || (res.data as any);+ return { concurrency: d.concurrency, maxConcurrency: d.maxConcurrency ?? d.max_concurrency };+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get concurrency");+ throw err;+ }+}++export async function getCreditUsage(http: HttpClient): Promise<CreditUsage> {+ try {+ const res = await http.get<{ success: boolean; data?: { remaining_credits: number } }>("/v2/team/credit-usage");+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get credit usage");+ const d = res.data.data || (res.data as any);+ return { remaining_credits: d.remaining_credits };+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get credit usage");+ throw err;+ }+}++export async function getTokenUsage(http: HttpClient): Promise<TokenUsage> {+ try {+ const res = await http.get<{ success: boolean; data?: TokenUsage }>("/v2/team/token-usage");+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get token usage");+ return (res.data.data || (res.data as any)) as TokenUsage;+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get token usage");+ throw err;+ }+}+diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsnew file mode 100644index 000000000..5dccff574--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -0,0 +1,305 @@+import type { ZodTypeAny } from "zod";+// Public types for Firecrawl JS/TS SDK v2 (camelCase only)++export type FormatString =+ | "markdown"+ | "html"+ | "rawHtml"+ | "links"+ | "screenshot"+ | "summary"+ | "changeTracking"+ | "json";++export interface Viewport {+ width: number;+ height: number;+}++export interface Format {+ type: FormatString;+}++export interface JsonFormat extends Format {+ type: "json";+ prompt: string;+ schema: Record<string, unknown> | ZodTypeAny;+}++export interface ScreenshotFormat {+ type: "screenshot";+ fullPage?: boolean;+ quality?: number;+ viewport?: Viewport | { width: number; height: number };+}++export interface ChangeTrackingFormat extends Format {+ type: "changeTracking";+ modes: ("git-diff" | "json")[];+ schema?: Record<string, unknown>;+ prompt?: string;+ tag?: string;+}++export type FormatOption =+ | FormatString+ | Format+ | JsonFormat+ | ChangeTrackingFormat+ | ScreenshotFormat;++export interface LocationConfig {+ country?: string;+ languages?: string[];+}++export interface WaitAction {+ type: "wait";+ milliseconds?: number;+ selector?: string;+}++export interface ScreenshotAction {+ type: "screenshot";+ fullPage?: boolean;+ quality?: number;+ viewport?: Viewport | { width: number; height: number };+}++export interface ClickAction {+ type: "click";+ selector: string;+}++export interface WriteAction {+ type: "write";+ text: string;+}++export interface PressAction {+ type: "press";+ key: string;+}++export interface ScrollAction {+ type: "scroll";+ direction: "up" | "down";+ selector?: string;+}++export interface ScrapeAction {+ type: "scrape";+}++export interface ExecuteJavascriptAction {+ type: "executeJavascript";+ script: string;+}++export interface PDFAction {+ type: "pdf";+ format?: "A0" | "A1" | "A2" | "A3" | "A4" | "A5" | "A6" | "Letter" | "Legal" | "Tabloid" | "Ledger";+ landscape?: boolean;+ scale?: number;+}++export type ActionOption =+ | WaitAction+ | ScreenshotAction+ | ClickAction+ | WriteAction+ | PressAction+ | ScrollAction+ | ScrapeAction+ | ExecuteJavascriptAction+ | PDFAction;++export interface ScrapeOptions {+ formats?: FormatOption[];+ headers?: Record<string, string>;+ includeTags?: string[];+ excludeTags?: string[];+ onlyMainContent?: boolean;+ timeout?: number;+ waitFor?: number;+ mobile?: boolean;+ parsers?: string[];+ actions?: ActionOption[];+ location?: LocationConfig;+ skipTlsVerification?: boolean;+ removeBase64Images?: boolean;+ fastMode?: boolean;+ useMock?: string;+ blockAds?: boolean;+ proxy?: "basic" | "stealth" | "auto" | string;+ maxAge?: number;+ storeInCache?: boolean;+}++export interface WebhookConfig {+ url: string;+ headers?: Record<string, string>;+ metadata?: Record<string, string>;+ events?: Array<"completed" | "failed" | "page" | "started">;+}++export interface DocumentMetadata {+ title?: string;+ description?: string;+ language?: string;+ keywords?: string | string[];+ robots?: string;+ ogTitle?: string;+ ogDescription?: string;+ ogUrl?: string;+ ogImage?: string;+ sourceURL?: string;+ statusCode?: number;+ error?: string;+ [key: string]: unknown;+}++export interface Document {+ markdown?: string;+ html?: string;+ rawHtml?: string;+ json?: unknown;+ metadata?: DocumentMetadata;+ links?: string[];+ screenshot?: string;+ actions?: Record<string, unknown>;+ warning?: string;+ changeTracking?: Record<string, unknown>;+}++export interface SearchResult {+ url: string;+ title?: string;+ description?: string;+}++export interface SearchData {+ web?: Array<SearchResult>;+ news?: Array<SearchResult>;+ images?: Array<SearchResult>;+}++export interface SearchRequest {+ query: string;+ sources?: Array<"web" | "news" | "images" | { type: "web" | "news" | "images" }>;+ limit?: number;+ tbs?: string;+ location?: string;+ ignoreInvalidURLs?: boolean;+ timeout?: number; // ms+ scrapeOptions?: ScrapeOptions;+}++export interface CrawlResponse {+ id: string;+ url: string;+}++export interface CrawlJob {+ status: "scraping" | "completed" | "failed" | "cancelled";+ total: number;+ completed: number;+ creditsUsed?: number;+ expiresAt?: string;+ next?: string | null;+ data: Document[];+}++export interface BatchScrapeResponse {+ id: string;+ url: string;+ invalidURLs?: string[];+}++export interface BatchScrapeJob {+ status: "scraping" | "completed" | "failed" | "cancelled";+ completed: number;+ total: number;+ creditsUsed?: number;+ expiresAt?: string;+ next?: string | null;+ data: Document[];+}++export interface MapData {+ links: SearchResult[];+}++export interface MapOptions {+ search?: string;+ sitemap?: "only" | "include" | "skip";+ includeSubdomains?: boolean;+ limit?: number;+ timeout?: number;+}++export interface ExtractResponse {+ success?: boolean;+ id?: string;+ status?: "processing" | "completed" | "failed" | "cancelled";+ data?: unknown;+ error?: string;+ warning?: string;+ sources?: Record<string, unknown>;+ expiresAt?: string;+}++export interface ConcurrencyCheck {+ concurrency: number;+ maxConcurrency: number;+}++export interface CreditUsage {+ remaining_credits: number;+}++export interface TokenUsage {+ promptTokens: number;+ completionTokens: number;+ totalTokens: number;+ step?: string;+ model?: string;+}++export interface CrawlErrorsResponse {+ errors: Array<Record<string, string>>;+ robotsBlocked: string[];+}++export interface ActiveCrawl {+ id: string;+ teamId: string;+ url: string;+ options?: Record<string, unknown> | null;+}++export interface ActiveCrawlsResponse {+ success: boolean;+ crawls: ActiveCrawl[];+}++export interface ErrorDetails {+ code?: string;+ message: string;+ details?: Record<string, unknown>;+ status?: number;+}++export class SdkError extends Error {+ status?: number;+ code?: string;+ details?: unknown;+ constructor(message: string, status?: number, code?: string, details?: unknown) {+ super(message);+ this.name = "FirecrawlSdkError";+ this.status = status;+ this.code = code;+ this.details = details;+ }+}+diff --git a/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.tsnew file mode 100644index 000000000..10adfc974--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/utils/errorHandler.ts@@ -0,0 +1,18 @@+import { type AxiosError, type AxiosResponse } from "axios";+import { SdkError } from "../types";++export function throwForBadResponse(resp: AxiosResponse, action: string): never {+ const status = resp.status;+ const body = resp.data || {};+ const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;+ throw new SdkError(msg, status, undefined, body?.details);+}++export function normalizeAxiosError(err: AxiosError, action: string): never {+ const status = err.response?.status;+ const body: any = err.response?.data;+ const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;+ const code = (body?.code as string) || err.code;+ throw new SdkError(message, status, code, body?.details ?? body);+}+diff --git a/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.tsnew file mode 100644index 000000000..b646e9316--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/utils/getVersion.ts@@ -0,0 +1,10 @@+export function getVersion(): string {+ try {+ // eslint-disable-next-line @typescript-eslint/no-var-requires+ const pkg = require("../../package.json");+ return (pkg?.version as string) || "3.0.0";+ } catch {+ return "3.0.0";+ }+}+diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.tsnew file mode 100644index 000000000..b2e57b487--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts@@ -0,0 +1,103 @@+import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse } from "axios";+import { getVersion } from "./getVersion";++export interface HttpClientOptions {+ apiKey: string;+ apiUrl: string;+ timeoutMs?: number;+ maxRetries?: number;+ backoffFactor?: number; // seconds factor for 0.5, 1, 2...+}++export class HttpClient {+ private instance: AxiosInstance;+ private readonly apiKey: string;+ private readonly apiUrl: string;+ private readonly maxRetries: number;+ private readonly backoffFactor: number;++ constructor(options: HttpClientOptions) {+ this.apiKey = options.apiKey;+ this.apiUrl = options.apiUrl.replace(/\/$/, "");+ this.maxRetries = options.maxRetries ?? 3;+ this.backoffFactor = options.backoffFactor ?? 0.5;+ this.instance = axios.create({+ baseURL: this.apiUrl,+ timeout: options.timeoutMs ?? 60000,+ headers: {+ "Content-Type": "application/json",+ Authorization: `Bearer ${this.apiKey}`,+ },+ transitional: { clarifyTimeoutError: true },+ });+ }++ getApiUrl(): string {+ return this.apiUrl;+ }++ getApiKey(): string {+ return this.apiKey;+ }++ private async request<T = any>(config: AxiosRequestConfig): Promise<AxiosResponse<T>> {+ const version = getVersion();+ config.headers = {+ ...(config.headers || {}),+ // origin header for GET/DELETE; for POST we also include in body below+ origin: `js-sdk@${version}`,+ };++ console.log("request", config);++ let lastError: any;+ for (let attempt = 0; attempt < this.maxRetries; attempt++) {+ try {+ const cfg: AxiosRequestConfig = { ...config };+ // For POST/PUT, ensure origin is present in JSON body too+ if (cfg.method && ["post", "put", "patch"].includes(cfg.method.toLowerCase())) {+ const data = (cfg.data ?? {}) as Record<string, unknown>;+ cfg.data = { ...data, origin: `js-sdk@${version}` };+ }+ const res = await this.instance.request<T>(cfg);+ if (res.status === 502 && attempt < this.maxRetries - 1) {+ await this.sleep(this.backoffFactor * Math.pow(2, attempt));+ continue;+ }+ return res;+ } catch (err: any) {+ lastError = err;+ const status = err?.response?.status;+ if (status === 502 && attempt < this.maxRetries - 1) {+ await this.sleep(this.backoffFactor * Math.pow(2, attempt));+ continue;+ }+ throw err;+ }+ }+ throw lastError ?? new Error("Unexpected HTTP client error");+ }++ private sleep(seconds: number): Promise<void> {+ return new Promise((r) => setTimeout(r, seconds * 1000));+ }++ post<T = any>(endpoint: string, body: Record<string, unknown>, headers?: Record<string, string>) {+ return this.request<T>({ method: "post", url: endpoint, data: body, headers });+ }++ get<T = any>(endpoint: string, headers?: Record<string, string>) {+ return this.request<T>({ method: "get", url: endpoint, headers });+ }++ delete<T = any>(endpoint: string, headers?: Record<string, string>) {+ return this.request<T>({ method: "delete", url: endpoint, headers });+ }++ prepareHeaders(idempotencyKey?: string): Record<string, string> {+ const headers: Record<string, string> = {};+ if (idempotencyKey) headers["x-idempotency-key"] = idempotencyKey;+ return headers;+ }+}+diff --git a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts b/apps/js-sdk/firecrawl/src/v2/utils/validation.tsnew file mode 100644index 000000000..82c97ee35--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts@@ -0,0 +1,50 @@+import { type FormatOption, type JsonFormat, type ScrapeOptions, type ScreenshotFormat } from "../types";+import zodToJsonSchema from "zod-to-json-schema";++export function ensureValidFormats(formats?: FormatOption[]): void {+ if (!formats) return;+ for (const fmt of formats) {+ if (typeof fmt === "string") {+ if (fmt === "json") {+ throw new Error("json format must be an object with { type: 'json', prompt, schema }");+ }+ continue;+ }+ if ((fmt as JsonFormat).type === "json") {+ const j = fmt as JsonFormat;+ if (!j.prompt || !j.schema) {+ throw new Error("json format requires 'prompt' and 'schema'");+ }+ // Flexibility: allow passing a Zod schema. Convert to JSON schema internally.+ const maybeSchema: any = j.schema as any;+ const isZod = !!maybeSchema && (typeof maybeSchema.safeParse === "function" || typeof maybeSchema.parse === "function") && !!maybeSchema._def;+ if (isZod) {+ try {+ (j as any).schema = zodToJsonSchema(maybeSchema);+ } catch {+ // If conversion fails, leave as-is; server-side may still handle, or request will fail explicitly+ }+ }+ continue;+ }+ if ((fmt as ScreenshotFormat).type === "screenshot") {+ // no-op; already camelCase; validate numeric fields if present+ const s = fmt as ScreenshotFormat;+ if (s.quality != null && (typeof s.quality !== "number" || s.quality < 0)) {+ throw new Error("screenshot.quality must be a non-negative number");+ }+ }+ }+}++export function ensureValidScrapeOptions(options?: ScrapeOptions): void {+ if (!options) return;+ if (options.timeout != null && options.timeout <= 0) {+ throw new Error("timeout must be positive");+ }+ if (options.waitFor != null && options.waitFor < 0) {+ throw new Error("waitFor must be non-negative");+ }+ ensureValidFormats(options.formats);+}+diff --git a/apps/js-sdk/firecrawl/src/v2/watcher.ts b/apps/js-sdk/firecrawl/src/v2/watcher.tsnew file mode 100644index 000000000..2e595c20d--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/watcher.ts@@ -0,0 +1,153 @@+import { EventEmitter } from "events";+import type { BatchScrapeJob, CrawlJob, Document } from "./types";+import type { HttpClient } from "./utils/httpClient";+import { getBatchScrapeStatus } from "./methods/batch";+import { getCrawlStatus } from "./methods/crawl";++type JobKind = "crawl" | "batch";++export interface WatcherOptions {+ kind?: JobKind;+ pollInterval?: number; // seconds+ timeout?: number; // seconds+}++type Snapshot = CrawlJob | BatchScrapeJob;++export class Watcher extends EventEmitter {+ private readonly http: HttpClient;+ private readonly jobId: string;+ private readonly kind: JobKind;+ private readonly pollInterval: number;+ private readonly timeout?: number;+ private ws?: WebSocket;+ private closed = false;++ constructor(http: HttpClient, jobId: string, opts: WatcherOptions = {}) {+ super();+ this.http = http;+ this.jobId = jobId;+ this.kind = opts.kind ?? "crawl";+ this.pollInterval = opts.pollInterval ?? 2;+ this.timeout = opts.timeout;+ }++ private buildWsUrl(): string {+ // replace http/https with ws/wss+ const apiUrl = this.http.getApiUrl();+ const wsBase = apiUrl.replace(/^http/, "ws");+ const path = this.kind === "crawl" ? `/v2/crawl/${this.jobId}` : `/v2/batch/scrape/${this.jobId}`;+ return `${wsBase}${path}`;+ }++ async start(): Promise<void> {+ try {+ const url = this.buildWsUrl();+ // Pass API key as subprotocol for browser compatibility+ this.ws = new WebSocket(url, this.http.getApiKey());+ this.attachWsHandlers(this.ws);+ } catch {+ // Fallback to polling immediately+ this.pollLoop();+ }+ }++ private attachWsHandlers(ws: WebSocket) {+ let startTs = Date.now();+ const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;+ ws.onmessage = (ev: MessageEvent) => {+ try {+ const body = typeof ev.data === "string" ? JSON.parse(ev.data) : null;+ if (!body) return;+ const type = body.type as string | undefined;+ if (type === "error") {+ this.emit("error", { status: "failed", data: [], error: body.error, id: this.jobId });+ return;+ }+ if (type === "catchup") {+ const payload = body.data || {};+ this.emitDocuments(payload.data || []);+ this.emitSnapshot(payload);+ return;+ }+ if (type === "document") {+ const doc = body.data;+ if (doc) this.emit("document", doc as Document & { id: string });+ return;+ }+ if (type === "done") {+ this.emit("done", { status: "completed", data: [], id: this.jobId });+ this.close();+ return;+ }+ const payload = body.data || body;+ if (payload && payload.status) this.emitSnapshot(payload);+ } catch {+ // ignore+ }+ if (timeoutMs && Date.now() - startTs > timeoutMs) this.close();+ };+ ws.onerror = () => {+ this.emit("error", { status: "failed", data: [], error: "WebSocket error", id: this.jobId });+ this.close();+ };+ ws.onclose = () => {+ if (!this.closed) this.pollLoop();+ };+ }++ private emitDocuments(docs: Document[]) {+ for (const doc of docs) this.emit("document", { ...(doc as any), id: this.jobId });+ }++ private emitSnapshot(payload: any) {+ const status = payload.status as Snapshot["status"];+ const data = (payload.data || []) as Document[];+ const snap: Snapshot = this.kind === "crawl"+ ? {+ status,+ completed: payload.completed ?? 0,+ total: payload.total ?? 0,+ creditsUsed: payload.creditsUsed,+ expiresAt: payload.expiresAt,+ next: payload.next ?? null,+ data,+ }+ : {+ status,+ completed: payload.completed ?? 0,+ total: payload.total ?? 0,+ creditsUsed: payload.creditsUsed,+ expiresAt: payload.expiresAt,+ next: payload.next ?? null,+ data,+ };+ this.emit("snapshot", snap);+ if (["completed", "failed", "cancelled"].includes(status)) this.emit("done", { status, data, id: this.jobId });+ }++ private async pollLoop() {+ const startTs = Date.now();+ const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;+ while (!this.closed) {+ try {+ const snap = this.kind === "crawl" ? await getCrawlStatus(this as any as HttpClient, this.jobId) : await getBatchScrapeStatus(this as any as HttpClient, this.jobId);+ this.emit("snapshot", snap);+ if (["completed", "failed", "cancelled"].includes(snap.status)) {+ this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });+ break;+ }+ } catch {+ // ignore polling errors+ }+ if (timeoutMs && Date.now() - startTs > timeoutMs) break;+ await new Promise((r) => setTimeout(r, Math.max(1000, this.pollInterval * 1000)));+ }+ }++ close() {+ this.closed = true;+ if (this.ws && (this.ws as any).close) (this.ws as any).close();+ }+}+diff --git a/apps/js-sdk/firecrawl/tsconfig.json b/apps/js-sdk/firecrawl/tsconfig.jsonindex 1297aed94..dbfc40ca2 100644--- a/apps/js-sdk/firecrawl/tsconfig.json+++ b/apps/js-sdk/firecrawl/tsconfig.json@@ -17,7 +17,8 @@"noImplicitOverride": true,/* If NOT transpiling with TypeScript: */- "module": "NodeNext",+ "module": "ESNext",+ "moduleResolution": "Bundler","noEmit": true,},"include": ["src/**/*"],diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.pyindex 68a08aa3e..a1468d44e 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py@@ -38,11 +38,12 @@ def test_search_minimal_request():assert isinstance(results, SearchData)assert hasattr(results, 'web')- assert hasattr(results, 'news')- assert hasattr(results, 'images')-assert results.web is not Noneassert len(results.web) > 0+ assert hasattr(results, 'news')+ assert results.news is None+ assert hasattr(results, 'images')+ assert results.images is Nonefor result in results.web:assert isinstance(result, SearchResult)