Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(js-sdk) fix: ws 'document' event implementation (#2415)

Rafael Millere03aa4f6e0e6
diff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.json
index 397dd0a9e..593cc4eaf 100644
--- a/apps/js-sdk/firecrawl/package-lock.json
+++ b/apps/js-sdk/firecrawl/package-lock.json
@@ -21,6 +21,7 @@
"@types/mocha": "^10.0.6",
"@types/node": "^20.12.12",
"@types/uuid": "^9.0.8",
+ "@types/ws": "^8.18.1",
"dotenv": "^16.4.5",
"jest": "^30.0.5",
"ts-jest": "^29.4.0",
@@ -1999,6 +2000,16 @@
"integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==",
"dev": true
},
+ "node_modules/@types/ws": {
+ "version": "8.18.1",
+ "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz",
+ "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "@types/node": "*"
+ }
+ },
"node_modules/@types/yargs": {
"version": "17.0.33",
"resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz",
@@ -5952,6 +5963,27 @@
"url": "https://github.com/sponsors/isaacs"
}
},
+ "node_modules/ws": {
+ "version": "8.18.3",
+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz",
+ "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=10.0.0"
+ },
+ "peerDependencies": {
+ "bufferutil": "^4.0.1",
+ "utf-8-validate": ">=5.0.2"
+ },
+ "peerDependenciesMeta": {
+ "bufferutil": {
+ "optional": true
+ },
+ "utf-8-validate": {
+ "optional": true
+ }
+ }
+ },
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index f700ac4c7..8deaa5081 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.5.0",
+ "version": "4.5.1",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
@@ -28,6 +28,7 @@
"dependencies": {
"axios": "^1.12.2",
"typescript-event-target": "^1.1.1",
+ "ws": "^8.18.3",
"zod": "^3.23.8",
"zod-to-json-schema": "^3.23.0"
},
@@ -42,6 +43,7 @@
"@types/mocha": "^10.0.6",
"@types/node": "^20.12.12",
"@types/uuid": "^9.0.8",
+ "@types/ws": "^8.18.1",
"dotenv": "^16.4.5",
"jest": "^30.0.5",
"ts-jest": "^29.4.0",
diff --git a/apps/js-sdk/firecrawl/pnpm-lock.yaml b/apps/js-sdk/firecrawl/pnpm-lock.yaml
index 5198627b9..45b7bd522 100644
--- a/apps/js-sdk/firecrawl/pnpm-lock.yaml
+++ b/apps/js-sdk/firecrawl/pnpm-lock.yaml
@@ -19,6 +19,9 @@ importers:
typescript-event-target:
specifier: ^1.1.1
version: 1.1.1
+ ws:
+ specifier: ^8.18.3
+ version: 8.18.3
zod:
specifier: ^3.23.8
version: 3.25.76
@@ -44,6 +47,9 @@ importers:
'@types/uuid':
specifier: ^9.0.8
version: 9.0.8
+ '@types/ws':
+ specifier: ^8.18.1
+ version: 8.18.1
dotenv:
specifier: ^16.4.5
version: 16.6.1
@@ -672,6 +678,9 @@ packages:
resolution: {integrity: sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}
+
resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==}
@@ -1902,6 +1911,18 @@ packages:
resolution: {integrity: sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
+ resolution: {integrity: sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==}
+ engines: {node: '>=10.0.0'}
+ peerDependencies:
+ bufferutil: ^4.0.1
+ utf-8-validate: '>=5.0.2'
+ peerDependenciesMeta:
+ bufferutil:
+ optional: true
+ utf-8-validate:
+ optional: true
+
resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==}
engines: {node: '>=10'}
@@ -2570,6 +2591,10 @@ snapshots:
'@types/[email protected]': {}
+ '@types/[email protected]':
+ dependencies:
+ '@types/node': 20.19.9
+
'@types/[email protected]': {}
@@ -3923,6 +3948,8 @@ snapshots:
imurmurhash: 0.1.4
signal-exit: 4.1.0
+
diff --git a/apps/js-sdk/firecrawl/src/v2/watcher.ts b/apps/js-sdk/firecrawl/src/v2/watcher.ts
index f64e610f9..f01e87a9b 100644
--- a/apps/js-sdk/firecrawl/src/v2/watcher.ts
+++ b/apps/js-sdk/firecrawl/src/v2/watcher.ts
@@ -3,6 +3,7 @@ import type { BatchScrapeJob, CrawlJob, Document } from "./types";
import type { HttpClient } from "./utils/httpClient";
import { getBatchScrapeStatus } from "./methods/batch";
import { getCrawlStatus } from "./methods/crawl";
+import { WebSocket as WS } from "ws";
type JobKind = "crawl" | "batch";
@@ -43,11 +44,17 @@ export class Watcher extends EventEmitter {
async start(): Promise<void> {
try {
const url = this.buildWsUrl();
- // Pass API key as subprotocol for browser compatibility
- this.ws = new WebSocket(url, this.http.getApiKey());
- this.attachWsHandlers(this.ws);
- } catch {
- // Fallback to polling immediately
+
+ if (typeof WebSocket !== 'undefined') {
+ this.ws = new WebSocket(url, this.http.getApiKey()) as any;
+ } else {
+ this.ws = new WS(url, this.http.getApiKey()) as any;
+ }
+
+ if (this.ws) {
+ this.attachWsHandlers(this.ws);
+ }
+ } catch (err) {
this.pollLoop();
}
}
@@ -76,7 +83,9 @@ export class Watcher extends EventEmitter {
return;
}
if (type === "done") {
- this.emit("done", { status: "completed", data: [], id: this.jobId });
+ const payload = body.data || body;
+ const data = (payload.data || []) as Document[];
+ this.emit("done", { status: "completed", data, id: this.jobId });
this.close();
return;
}
diff --git a/apps/js-sdk/firecrawl/tsup.config.ts b/apps/js-sdk/firecrawl/tsup.config.ts
index 31d677392..632f7b019 100644
--- a/apps/js-sdk/firecrawl/tsup.config.ts
+++ b/apps/js-sdk/firecrawl/tsup.config.ts
@@ -8,6 +8,7 @@ export default defineConfig({
clean: true,
platform: "node",
target: "node22",
+ external: ["ws"],
noExternal: ["typescript-event-target"],
esbuildOptions(options) {
options.define = {