Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(js-sdk): load in browser bundles and on Node 18 and 20 (#5242)
Erik Engervallfc74e9034699
diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.mdindex a98cdf06a..fed0e7afd 100644--- a/apps/js-sdk/firecrawl/README.md+++ b/apps/js-sdk/firecrawl/README.md@@ -15,7 +15,7 @@ npm install firecrawl1. Get an API key from [firecrawl.dev](https://firecrawl.dev)2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as the `apiKey` option to the `Firecrawl` class.-Here's an example of how to use the SDK with error handling:+Here's an example of how to use the SDK with error handling. It uses ES modules and top-level `await`, so save it as an `.mjs` file or set `"type": "module"` in your `package.json`:```jsimport { Firecrawl } from 'firecrawl';diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 6a9a918a4..61379075d 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.49.2",+ "version": "4.50.0","description": "JavaScript SDK for Firecrawl, the context API for AI agents: search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and search scientific papers (PubMed, bioRxiv, medRxiv, arXiv)","main": "dist/index.js","types": "dist/index.d.ts",@@ -70,6 +70,6 @@"bioinformatics"],"engines": {- "node": ">=22.0.0"+ "node": ">=18.0.0"}}diff --git a/apps/js-sdk/firecrawl/src/types/node-undici.d.ts b/apps/js-sdk/firecrawl/src/types/node-undici.d.tsdeleted file mode 100644index f51b51d28..000000000--- a/apps/js-sdk/firecrawl/src/types/node-undici.d.ts+++ /dev/null@@ -1,10 +0,0 @@-type WebSocketCtor = typeof globalThis.WebSocket;--declare module "node:undici" {- export const WebSocket: WebSocketCtor;- const _default: {- WebSocket: WebSocketCtor;- };- export default _default;-}-diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex 2661a9fb8..02f56d7d2 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -686,7 +686,7 @@ export default class FirecrawlApp {return packageJson.default.version;} catch (error) {// Suppress noisy logs under test environments- const isTest = typeof process !== 'undefined' && process.env.NODE_ENV === 'test';+ const isTest = typeof process !== 'undefined' && !!process.env && process.env.NODE_ENV === 'test';if (!isTest) {// eslint-disable-next-line no-consoleconsole.error("Error getting version:", error);@@ -2220,6 +2220,9 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {this.id = id;// replace `http` with `ws` (`http://` -> `ws://` and `https://` -> `wss://`)const wsUrl = app.apiUrl.replace(/^http/, "ws");+ if (typeof WebSocket === "undefined") {+ throw new Error("Watching a crawl needs a global WebSocket (Node 22+, Bun, Deno or a browser). On older Node, use the v2 client's watcher(), which polls instead.");+ }this.ws = new WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);this.status = "scraping";this.data = [];diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex 9aba2bdc7..0038d5eb4 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -164,8 +164,9 @@ export class FirecrawlClient {const opts: FirecrawlClientOptions =typeof options === "string" ? { apiKey: options } : options;- const apiKey = (opts.apiKey ?? process.env.FIRECRAWL_API_KEY ?? "").trim();- const apiUrl = (opts.apiUrl ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev").replace(/\/$/, "");+ const hasEnv = typeof process !== "undefined" && !!process.env;+ const apiKey = (opts.apiKey ?? (hasEnv ? process.env.FIRECRAWL_API_KEY : undefined) ?? "").trim();+ const apiUrl = (opts.apiUrl ?? (hasEnv ? process.env.FIRECRAWL_API_URL : undefined) ?? "https://api.firecrawl.dev").replace(/\/$/, "");// No API key is allowed: scrape, search, and interact fall back to the// keyless free tier (rate-limited per IP). Other methods will return 401diff --git a/apps/js-sdk/firecrawl/src/v2/watcher.ts b/apps/js-sdk/firecrawl/src/v2/watcher.tsindex c021a687e..6824c48c8 100644--- a/apps/js-sdk/firecrawl/src/v2/watcher.ts+++ b/apps/js-sdk/firecrawl/src/v2/watcher.ts@@ -1,48 +1,110 @@-import { EventEmitter } from "events";import type { BatchScrapeJob, CrawlJob, Document } from "./types";import type { HttpClient } from "./utils/httpClient";import { getBatchScrapeStatus } from "./methods/batch";import { getCrawlStatus } from "./methods/crawl";-// Note: browsers/Deno expose globalThis.WebSocket, but many Node runtimes (<22.4 or without-// experimental flags) do not. We lazily fall back to node:undici.type WebSocketConstructor = new (url: string, protocols?: string | string[]) => WebSocket;-const hasGlobalWebSocket = (): WebSocketConstructor | undefined => {- if (typeof globalThis === "undefined") return undefined;+// Runtimes without a global WebSocket (Node before 22) poll instead.+const getWebSocketCtor = (): WebSocketConstructor | undefined => {const candidate = (globalThis as any).WebSocket;return typeof candidate === "function" ? (candidate as WebSocketConstructor) : undefined;};-const isNodeRuntime = () => typeof process !== "undefined" && !!process.versions?.node;+type Listener = ((...args: any[]) => void) & { listener?: Listener };-let cachedWebSocket: WebSocketConstructor | undefined;-let loadPromise: Promise<WebSocketConstructor | undefined> | undefined;+/** Node's EventEmitter API for the watcher, with no import of "events", so browser bundles load it. */+class Emitter {+ private readonly listenersByEvent = new Map<string | symbol, Listener[]>();+ private maxListeners = 10;-const loadNodeWebSocket = async (): Promise<WebSocketConstructor | undefined> => {- if (!isNodeRuntime()) return undefined;- try {- const undici = await import("node:undici");- const ctor = (undici as any).WebSocket ?? (undici as any).default?.WebSocket;- return typeof ctor === "function" ? (ctor as WebSocketConstructor) : undefined;- } catch {- return undefined;+ on(event: string | symbol, listener: Listener): this {+ this.listenersByEvent.set(event, [...(this.listenersByEvent.get(event) ?? []), listener]);+ return this;}-};-const getWebSocketCtor = async (): Promise<WebSocketConstructor | undefined> => {- if (cachedWebSocket) return cachedWebSocket;- const globalWs = hasGlobalWebSocket();- if (globalWs) {- cachedWebSocket = globalWs;- return cachedWebSocket;+ prependListener(event: string | symbol, listener: Listener): this {+ this.listenersByEvent.set(event, [listener, ...(this.listenersByEvent.get(event) ?? [])]);+ return this;}- if (!loadPromise) {- loadPromise = loadNodeWebSocket();++ prependOnceListener(event: string | symbol, listener: Listener): this {+ return this.prependListener(event, this.onceWrapper(event, listener));}- cachedWebSocket = await loadPromise;- return cachedWebSocket;-};++ listeners(event: string | symbol): Listener[] {+ return (this.listenersByEvent.get(event) ?? []).map((entry) => entry.listener ?? entry);+ }++ rawListeners(event: string | symbol): Listener[] {+ return [...(this.listenersByEvent.get(event) ?? [])];+ }++ eventNames(): (string | symbol)[] {+ return [...this.listenersByEvent.keys()].filter((event) => this.listenerCount(event) > 0);+ }++ setMaxListeners(n: number): this {+ this.maxListeners = n;+ return this;+ }++ getMaxListeners(): number {+ return this.maxListeners;+ }++ addListener(event: string | symbol, listener: Listener): this {+ return this.on(event, listener);+ }++ once(event: string | symbol, listener: Listener): this {+ return this.on(event, this.onceWrapper(event, listener));+ }++ private onceWrapper(event: string | symbol, listener: Listener): Listener {+ const wrapped: Listener = (...args) => {+ this.off(event, wrapped);+ listener.apply(this, args);+ };+ wrapped.listener = listener;+ return wrapped;+ }++ off(event: string | symbol, listener: Listener): this {+ const current = this.listenersByEvent.get(event);+ if (!current) return this;+ for (let index = current.length - 1; index >= 0; index--) {+ if (current[index] !== listener && current[index]!.listener !== listener) continue;+ this.listenersByEvent.set(event, current.filter((_, i) => i !== index));+ break;+ }+ return this;+ }++ removeListener(event: string | symbol, listener: Listener): this {+ return this.off(event, listener);+ }++ removeAllListeners(event?: string | symbol): this {+ if (event === undefined) this.listenersByEvent.clear();+ else this.listenersByEvent.delete(event);+ return this;+ }++ listenerCount(event: string | symbol): number {+ return this.listenersByEvent.get(event)?.length ?? 0;+ }++ emit(event: string | symbol, ...args: any[]): boolean {+ const current = this.listenersByEvent.get(event);+ if (!current?.length) {+ if (event === "error") throw args[0] instanceof Error ? args[0] : new Error(String(args[0]));+ return false;+ }+ for (const listener of [...current]) listener.apply(this, args);+ return true;+ }+}const decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : undefined;@@ -81,7 +143,7 @@ export interface WatcherOptions {type Snapshot = CrawlJob | BatchScrapeJob;-export class Watcher extends EventEmitter {+export class Watcher extends Emitter {private readonly http: HttpClient;private readonly jobId: string;private readonly kind: JobKind;@@ -122,7 +184,7 @@ export class Watcher extends EventEmitter {(async () => {try {const url = this.buildWsUrl();- const wsCtor = await getWebSocketCtor();+ const wsCtor = getWebSocketCtor();if (!wsCtor) {this.pollLoop();return;