Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(js sdk)fix/ws deno (#2476)

Rafael Millere4acbcd75ca7
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index e7215f00a..8b74e4f13 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.8.0",
+ "version": "4.8.1",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
@@ -28,7 +28,6 @@
"dependencies": {
"axios": "^1.12.2",
"typescript-event-target": "^1.1.1",
- "ws": "^8.18.3",
"zod": "^3.23.8",
"zod-to-json-schema": "^3.23.0"
},
@@ -43,7 +42,6 @@
"@types/mocha": "^10.0.6",
"@types/node": "^20.12.12",
"@types/uuid": "^9.0.8",
- "@types/ws": "^8.18.1",
"dotenv": "^16.4.5",
"jest": "^30.2.0",
"ts-jest": "^29.4.5",
diff --git a/apps/js-sdk/firecrawl/pnpm-lock.yaml b/apps/js-sdk/firecrawl/pnpm-lock.yaml
index 32f2d12f3..ef0b21060 100644
--- a/apps/js-sdk/firecrawl/pnpm-lock.yaml
+++ b/apps/js-sdk/firecrawl/pnpm-lock.yaml
@@ -49,9 +49,6 @@ importers:
'@types/uuid':
specifier: ^9.0.8
version: 9.0.8
- '@types/ws':
- specifier: ^8.18.1
- version: 8.18.1
dotenv:
specifier: ^16.4.5
version: 16.6.1
@@ -692,9 +689,6 @@ packages:
resolution: {integrity: sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==}
- '@types/[email protected]':
- resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}
-
resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==}
@@ -2604,10 +2598,6 @@ snapshots:
'@types/[email protected]': {}
- '@types/[email protected]':
- dependencies:
- '@types/node': 20.19.9
-
'@types/[email protected]': {}
diff --git a/apps/js-sdk/firecrawl/src/types/node-undici.d.ts b/apps/js-sdk/firecrawl/src/types/node-undici.d.ts
new file mode 100644
index 000000000..f51b51d28
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/types/node-undici.d.ts
@@ -0,0 +1,10 @@
+type WebSocketCtor = typeof globalThis.WebSocket;
+
+declare module "node:undici" {
+ export const WebSocket: WebSocketCtor;
+ const _default: {
+ WebSocket: WebSocketCtor;
+ };
+ export default _default;
+}
+
diff --git a/apps/js-sdk/firecrawl/src/v2/watcher.ts b/apps/js-sdk/firecrawl/src/v2/watcher.ts
index 1d64798a0..4f456d681 100644
--- a/apps/js-sdk/firecrawl/src/v2/watcher.ts
+++ b/apps/js-sdk/firecrawl/src/v2/watcher.ts
@@ -3,7 +3,73 @@ import type { BatchScrapeJob, CrawlJob, Document } from "./types";
import type { HttpClient } from "./utils/httpClient";
import { getBatchScrapeStatus } from "./methods/batch";
import { getCrawlStatus } from "./methods/crawl";
-import { WebSocket as WS } from "ws";
+// Note: browsers/Deno expose globalThis.WebSocket, but many Node runtimes (<22.4 or without
+// experimental flags) do not. We lazily fall back to node:undici.
+
+type WebSocketConstructor = new (url: string, protocols?: string | string[]) => WebSocket;
+
+const hasGlobalWebSocket = (): WebSocketConstructor | undefined => {
+ if (typeof globalThis === "undefined") return undefined;
+ const candidate = (globalThis as any).WebSocket;
+ return typeof candidate === "function" ? (candidate as WebSocketConstructor) : undefined;
+};
+
+const isNodeRuntime = () => typeof process !== "undefined" && !!process.versions?.node;
+
+let cachedWebSocket: WebSocketConstructor | undefined;
+let loadPromise: Promise<WebSocketConstructor | undefined> | undefined;
+
+const loadNodeWebSocket = async (): Promise<WebSocketConstructor | undefined> => {
+ if (!isNodeRuntime()) return undefined;
+ try {
+ const undici = await import("node:undici");
+ const ctor = (undici as any).WebSocket ?? (undici as any).default?.WebSocket;
+ return typeof ctor === "function" ? (ctor as WebSocketConstructor) : undefined;
+ } catch {
+ return undefined;
+ }
+};
+
+const getWebSocketCtor = async (): Promise<WebSocketConstructor | undefined> => {
+ if (cachedWebSocket) return cachedWebSocket;
+ const globalWs = hasGlobalWebSocket();
+ if (globalWs) {
+ cachedWebSocket = globalWs;
+ return cachedWebSocket;
+ }
+ if (!loadPromise) {
+ loadPromise = loadNodeWebSocket();
+ }
+ cachedWebSocket = await loadPromise;
+ return cachedWebSocket;
+};
+
+const decoder = typeof TextDecoder !== "undefined" ? new TextDecoder() : undefined;
+
+const ensureUtf8String = (data: unknown): string | undefined => {
+ if (typeof data === "string") return data;
+
+ if (typeof Buffer !== "undefined" && Buffer.isBuffer(data)) {
+ return data.toString("utf8");
+ }
+
+ const convertView = (view: ArrayBufferView): string | undefined => {
+ if (typeof Buffer !== "undefined") {
+ return Buffer.from(view.buffer, view.byteOffset, view.byteLength).toString("utf8");
+ }
+ return decoder?.decode(view);
+ };
+
+ if (ArrayBuffer.isView(data)) {
+ return convertView(data);
+ }
+
+ if (data instanceof ArrayBuffer) {
+ return convertView(new Uint8Array(data));
+ }
+
+ return undefined;
+};
type JobKind = "crawl" | "batch";
@@ -23,6 +89,7 @@ export class Watcher extends EventEmitter {
private readonly timeout?: number;
private ws?: WebSocket;
private closed = false;
+ private readonly emittedDocumentKeys = new Set<string>();
constructor(http: HttpClient, jobId: string, opts: WatcherOptions = {}) {
super();
@@ -44,11 +111,14 @@ export class Watcher extends EventEmitter {
async start(): Promise<void> {
try {
const url = this.buildWsUrl();
-
- if (typeof WebSocket !== 'undefined') {
- this.ws = new WebSocket(url, this.http.getApiKey()) as any;
- } else {
- this.ws = new WS(url, this.http.getApiKey()) as any;
+ const wsCtor = await getWebSocketCtor();
+ if (!wsCtor) {
+ this.pollLoop();
+ return;
+ }
+ this.ws = new wsCtor(url, this.http.getApiKey()) as any;
+ if (this.ws && "binaryType" in this.ws) {
+ (this.ws as any).binaryType = "arraybuffer";
}
if (this.ws) {
@@ -64,8 +134,9 @@ export class Watcher extends EventEmitter {
const timeoutMs = this.timeout ? this.timeout * 1000 : undefined;
ws.onmessage = (ev: MessageEvent) => {
try {
- const body = typeof ev.data === "string" ? JSON.parse(ev.data) : null;
- if (!body) return;
+ const raw = ensureUtf8String(ev.data);
+ if (!raw) return;
+ const body = JSON.parse(raw);
const type = body.type as string | undefined;
if (type === "error") {
this.emit("error", { status: "failed", data: [], error: body.error, id: this.jobId });
@@ -85,6 +156,7 @@ export class Watcher extends EventEmitter {
if (type === "done") {
const payload = body.data || body;
const data = (payload.data || []) as Document[];
+ if (data.length) this.emitDocuments(data);
this.emit("done", { status: "completed", data, id: this.jobId });
this.close();
return;
@@ -105,8 +177,28 @@ export class Watcher extends EventEmitter {
};
}
+ private documentKey(doc: Document): string {
+ if (doc && typeof doc === "object") {
+ const explicitId = (doc as any).id ?? (doc as any).docId ?? (doc as any).url;
+ if (typeof explicitId === "string" && explicitId.length) {
+ return explicitId;
+ }
+ }
+ try {
+ return JSON.stringify(doc);
+ } catch {
+ return `${Date.now()}-${Math.random()}`;
+ }
+ }
+
private emitDocuments(docs: Document[]) {
- for (const doc of docs) this.emit("document", { ...(doc as any), id: this.jobId });
+ for (const doc of docs) {
+ if (!doc) continue;
+ const key = this.documentKey(doc);
+ if (this.emittedDocumentKeys.has(key)) continue;
+ this.emittedDocumentKeys.add(key);
+ this.emit("document", { ...(doc as any), id: this.jobId });
+ }
}
private emitSnapshot(payload: any) {
@@ -148,6 +240,7 @@ export class Watcher extends EventEmitter {
const snap = this.kind === "crawl"
? await getCrawlStatus(this.http as any, this.jobId)
: await getBatchScrapeStatus(this.http as any, this.jobId);
+ this.emitDocuments((snap.data || []) as Document[]);
this.emit("snapshot", snap);
if (["completed", "failed", "cancelled"].includes(snap.status)) {
this.emit("done", { status: snap.status, data: snap.data, id: this.jobId });
diff --git a/apps/js-sdk/firecrawl/tsup.config.ts b/apps/js-sdk/firecrawl/tsup.config.ts
index 632f7b019..31d677392 100644
--- a/apps/js-sdk/firecrawl/tsup.config.ts
+++ b/apps/js-sdk/firecrawl/tsup.config.ts
@@ -8,7 +8,6 @@ export default defineConfig({
clean: true,
platform: "node",
target: "node22",
- external: ["ws"],
noExternal: ["typescript-event-target"],
esbuildOptions(options) {
options.define = {