Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(js sdk)fix/ws deno (#2476)
Rafael Millere4acbcd75ca7
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex e7215f00a..8b74e4f13 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.8.0",+ "version": "4.8.1","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",@@ -28,7 +28,6 @@"dependencies": {"axios": "^1.12.2","typescript-event-target": "^1.1.1",- "ws": "^8.18.3","zod": "^3.23.8","zod-to-json-schema": "^3.23.0"},@@ -43,7 +42,6 @@"@types/mocha": "^10.0.6","@types/node": "^20.12.12","@types/uuid": "^9.0.8",- "@types/ws": "^8.18.1","dotenv": "^16.4.5","jest": "^30.2.0","ts-jest": "^29.4.5",diff --git a/apps/js-sdk/firecrawl/pnpm-lock.yaml b/apps/js-sdk/firecrawl/pnpm-lock.yamlindex 32f2d12f3..ef0b21060 100644--- a/apps/js-sdk/firecrawl/pnpm-lock.yaml+++ b/apps/js-sdk/firecrawl/pnpm-lock.yaml@@ -49,9 +49,6 @@ importers:'@types/uuid':specifier: ^9.0.8version: 9.0.8- '@types/ws':- specifier: ^8.18.1- version: 8.18.1dotenv:specifier: ^16.4.5version: 16.6.1@@ -692,9 +689,6 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==}- '@types/[email protected]':- resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}-'@types/[email protected]':resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==}@@ -2604,10 +2598,6 @@ snapshots:'@types/[email protected]': {}- '@types/[email protected]':- dependencies:- '@types/node': 20.19.9-'@types/[email protected]': {}'@types/[email protected]':diff --git a/apps/js-sdk/firecrawl/src/types/node-undici.d.ts b/apps/js-sdk/firecrawl/src/types/node-undici.d.tsnew file mode 100644index 000000000..f51b51d28--- /dev/null+++ b/apps/js-sdk/firecrawl/src/types/node-undici.d.ts@@ -0,0 +1,10 @@+type WebSocketCtor = typeof globalThis.WebSocket;++declare module "node:undici" {+ export const WebSocket: WebSocketCtor;+ const _default: {+ WebSocket: WebSocketCtor;+ };+ export default _default;+}+diff --git a/apps/js-sdk/firecrawl/src/v2/watcher.ts b/apps/js-sdk/firecrawl/src/v2/watcher.tsindex 1d64798a0..4f456d681 100644--- a/apps/js-sdk/firecrawl/src/v2/watcher.ts+++ b/apps/js-sdk/firecrawl/src/v2/watcher.ts@@ -3,7 +3,73 @@ import type { BatchScrapeJob, CrawlJob, Document } from "./types";import type { HttpClient } from "./utils/httpClient";import { getBatchScrapeStatus } from "./methods/batch";import { getCrawlStatus } from "./methods/crawl";-import { WebSocket as WS } from "ws";+// Note: browsers/Deno expose globalThis.WebSocket, but many Node runtimes (<22.4 or without+// experimental flags) do not. We lazily fall back to node:undici.++type WebSocketConstructor = new (url: string, protocols?: string | string[]) => WebSocket;++const hasGlobalWebSocket = (): WebSocketConstructor | undefined => {+ if (typeof globalThis === "undefined") return undefined;+ const candidate = (globalThis as any).WebSocket;+ return typeof candidate === "function" ? (candidate as WebSocketConstructor) : undefined;+};++const isNodeRuntime = () => typeof process !== "undefined" && !!process.versions?.node;++let cachedWebSocket: WebSocketConstructor | undefined;+let loadPromise: Promise<WebSocketConstructor | undefined> | undefined;++const loadNodeWebSocket = async (): Promise<WebSocketConstructor | undefined> => {+ if (!isNodeRuntime()) return undefined;+ try {+ const undici = await import("node:undici");+ const ctor = (undici as any).WebSocket ?? (undici as any).default?.WebSocket;+ return typeof ctor === "function" ? (ctor as WebSocketConstructor) : undefined;+ } catch {+ return undefined;+ }+};++const getWebSocketCtor = async (): Promise<WebSocketConstructor | undefined> => {+ if (cachedWebSocket) return cachedWebSocket;+ const globalWs = hasGlobalWebSocket();+ if (globalWs) {+ cachedWebSocket = globalWs;+ return cachedWebSocket;+ }+ if (!loadPromise) {+ loadPromise = loadNodeWebSocket();+ }+ cachedWebSocket = await loadPromise;+ return cachedWebSocket;+};++const decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : undefined;++const ensureUtf8String = (data: unknown): string | undefined => {+ if (typeof data === "string") return data;++ if (typeof Buffer !== "undefined" && Buffer.isBuffer(data)) {+ return data.toString("utf8");+ }++ const convertView = (view: ArrayBufferView): string | undefined => {+ if (typeof Buffer !== "undefined") {+ return Buffer.from(view.buffer, view.byteOffset, view.byteLength).toString("utf8");+ }+ return decoder?.decode(view);+ };++ if (ArrayBuffer.isView(data)) {+ return convertView(data);+ }++ if (data instanceof ArrayBuffer) {+ return convertView(new Uint8Array(data));+ }++ return undefined;+};type JobKind = "crawl" | "batch";@@ -23,6 +89,7 @@ export class Watcher extends EventEmitter {private readonly timeout?: number;private ws?: WebSocket;private closed = false;+ private readonly emittedDocumentKeys = new Set<string>();constructor(http: HttpClient, jobId: string, opts: WatcherOptions = {}) {super();@@ -44,11 +111,14 @@ export class Watcher extends EventEmitter {async start(): Promise<void> {try {const url = this.buildWsUrl();-- if (typeof WebSocket !== 'undefined') {- this.ws = new WebSocket(url, this.http.getApiKey()) as any;- } else {- this.ws = new WS(url, this.http.getApiKey()) as any;+ const wsCtor = await getWebSocketCtor();+ if (!wsCtor) {+ this.pollLoop();+ return;+ }+ this.ws = new wsCtor(url, this.http.getApiKey()) as any;+ if (this.ws && "binaryType" in this.ws) {+ (this.ws as any).binaryType = "arraybuffer";}if (this.ws) {@@ -64,8 +134,9 @@ export class Watcher extends EventEmitter {const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;ws.onmessage = (ev: MessageEvent) => {try {- const body = typeof ev.data === "string" ? JSON.parse(ev.data) : null;- if (!body) return;+ const raw = ensureUtf8String(ev.data);+ if (!raw) return;+ const body = JSON.parse(raw);const type = body.type as string | undefined;if (type === "error") {this.emit("error", { status: "failed", data: [], error: body.error, id: this.jobId });@@ -85,6 +156,7 @@ export class Watcher extends EventEmitter {if (type === "done") {const payload = body.data || body;const data = (payload.data || []) as Document[];+ if (data.length) this.emitDocuments(data);this.emit("done", { status: "completed", data, id: this.jobId });this.close();return;@@ -105,8 +177,28 @@ export class Watcher extends EventEmitter {};}+ private documentKey(doc: Document): string {+ if (doc && typeof doc === "object") {+ const explicitId = (doc as any).id ?? (doc as any).docId ?? (doc as any).url;+ if (typeof explicitId === "string" && explicitId.length) {+ return explicitId;+ }+ }+ try {+ return JSON.stringify(doc);+ } catch {+ return `${Date.now()}-${Math.random()}`;+ }+ }+private emitDocuments(docs: Document[]) {- for (const doc of docs) this.emit("document", { ...(doc as any), id: this.jobId });+ for (const doc of docs) {+ if (!doc) continue;+ const key = this.documentKey(doc);+ if (this.emittedDocumentKeys.has(key)) continue;+ this.emittedDocumentKeys.add(key);+ this.emit("document", { ...(doc as any), id: this.jobId });+ }}private emitSnapshot(payload: any) {@@ -148,6 +240,7 @@ export class Watcher extends EventEmitter {const snap = this.kind === "crawl"? await getCrawlStatus(this.http as any, this.jobId): await getBatchScrapeStatus(this.http as any, this.jobId);+ this.emitDocuments((snap.data || []) as Document[]);this.emit("snapshot", snap);if (["completed", "failed", "cancelled"].includes(snap.status)) {this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });diff --git a/apps/js-sdk/firecrawl/tsup.config.ts b/apps/js-sdk/firecrawl/tsup.config.tsindex 632f7b019..31d677392 100644--- a/apps/js-sdk/firecrawl/tsup.config.ts+++ b/apps/js-sdk/firecrawl/tsup.config.ts@@ -8,7 +8,6 @@ export default defineConfig({clean: true,platform: "node",target: "node22",- external: ["ws"],noExternal: ["typescript-event-target"],esbuildOptions(options) {options.define = {