Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(js-sdk) fix: ws 'document' event implementation (#2415)
Rafael Millere03aa4f6e0e6
diff --git a/apps/js-sdk/firecrawl/package-lock.json b/apps/js-sdk/firecrawl/package-lock.jsonindex 397dd0a9e..593cc4eaf 100644--- a/apps/js-sdk/firecrawl/package-lock.json+++ b/apps/js-sdk/firecrawl/package-lock.json@@ -21,6 +21,7 @@"@types/mocha": "^10.0.6","@types/node": "^20.12.12","@types/uuid": "^9.0.8",+ "@types/ws": "^8.18.1","dotenv": "^16.4.5","jest": "^30.0.5","ts-jest": "^29.4.0",@@ -1999,6 +2000,16 @@"integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==","dev": true},+ "node_modules/@types/ws": {+ "version": "8.18.1",+ "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz",+ "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==",+ "dev": true,+ "license": "MIT",+ "dependencies": {+ "@types/node": "*"+ }+ },"node_modules/@types/yargs": {"version": "17.0.33","resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz",@@ -5952,6 +5963,27 @@"url": "https://github.com/sponsors/isaacs"}},+ "node_modules/ws": {+ "version": "8.18.3",+ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz",+ "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==",+ "license": "MIT",+ "engines": {+ "node": ">=10.0.0"+ },+ "peerDependencies": {+ "bufferutil": "^4.0.1",+ "utf-8-validate": ">=5.0.2"+ },+ "peerDependenciesMeta": {+ "bufferutil": {+ "optional": true+ },+ "utf-8-validate": {+ "optional": true+ }+ }+ },"node_modules/y18n": {"version": "5.0.8","resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex f700ac4c7..8deaa5081 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.5.0",+ "version": "4.5.1","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",@@ -28,6 +28,7 @@"dependencies": {"axios": "^1.12.2","typescript-event-target": "^1.1.1",+ "ws": "^8.18.3","zod": "^3.23.8","zod-to-json-schema": "^3.23.0"},@@ -42,6 +43,7 @@"@types/mocha": "^10.0.6","@types/node": "^20.12.12","@types/uuid": "^9.0.8",+ "@types/ws": "^8.18.1","dotenv": "^16.4.5","jest": "^30.0.5","ts-jest": "^29.4.0",diff --git a/apps/js-sdk/firecrawl/pnpm-lock.yaml b/apps/js-sdk/firecrawl/pnpm-lock.yamlindex 5198627b9..45b7bd522 100644--- a/apps/js-sdk/firecrawl/pnpm-lock.yaml+++ b/apps/js-sdk/firecrawl/pnpm-lock.yaml@@ -19,6 +19,9 @@ importers:typescript-event-target:specifier: ^1.1.1version: 1.1.1+ ws:+ specifier: ^8.18.3+ version: 8.18.3zod:specifier: ^3.23.8version: 3.25.76@@ -44,6 +47,9 @@ importers:'@types/uuid':specifier: ^9.0.8version: 9.0.8+ '@types/ws':+ specifier: ^8.18.1+ version: 8.18.1dotenv:specifier: ^16.4.5version: 16.6.1@@ -672,6 +678,9 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==}+ '@types/[email protected]':+ resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==}+'@types/[email protected]':resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==}@@ -1902,6 +1911,18 @@ packages:resolution: {integrity: sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==}engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}+ resolution: {integrity: sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==}+ engines: {node: '>=10.0.0'}+ peerDependencies:+ bufferutil: ^4.0.1+ utf-8-validate: '>=5.0.2'+ peerDependenciesMeta:+ bufferutil:+ optional: true+ utf-8-validate:+ optional: true+resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==}engines: {node: '>=10'}@@ -2570,6 +2591,10 @@ snapshots:'@types/[email protected]': {}+ '@types/[email protected]':+ dependencies:+ '@types/node': 20.19.9+'@types/[email protected]': {}'@types/[email protected]':@@ -3923,6 +3948,8 @@ snapshots:imurmurhash: 0.1.4signal-exit: 4.1.0+ [email protected]: {}+diff --git a/apps/js-sdk/firecrawl/src/v2/watcher.ts b/apps/js-sdk/firecrawl/src/v2/watcher.tsindex f64e610f9..f01e87a9b 100644--- a/apps/js-sdk/firecrawl/src/v2/watcher.ts+++ b/apps/js-sdk/firecrawl/src/v2/watcher.ts@@ -3,6 +3,7 @@ import type { BatchScrapeJob, CrawlJob, Document } from "./types";import type { HttpClient } from "./utils/httpClient";import { getBatchScrapeStatus } from "./methods/batch";import { getCrawlStatus } from "./methods/crawl";+import { WebSocket as WS } from "ws";type JobKind = "crawl" | "batch";@@ -43,11 +44,17 @@ export class Watcher extends EventEmitter {async start(): Promise<void> {try {const url = this.buildWsUrl();- // Pass API key as subprotocol for browser compatibility- this.ws = new WebSocket(url, this.http.getApiKey());- this.attachWsHandlers(this.ws);- } catch {- // Fallback to polling immediately++ if (typeof WebSocket !== 'undefined') {+ this.ws = new WebSocket(url, this.http.getApiKey()) as any;+ } else {+ this.ws = new WS(url, this.http.getApiKey()) as any;+ }++ if (this.ws) {+ this.attachWsHandlers(this.ws);+ }+ } catch (err) {this.pollLoop();}}@@ -76,7 +83,9 @@ export class Watcher extends EventEmitter {return;}if (type === "done") {- this.emit("done", { status: "completed", data: [], id: this.jobId });+ const payload = body.data || body;+ const data = (payload.data || []) as Document[];+ this.emit("done", { status: "completed", data, id: this.jobId });this.close();return;}diff --git a/apps/js-sdk/firecrawl/tsup.config.ts b/apps/js-sdk/firecrawl/tsup.config.tsindex 31d677392..632f7b019 100644--- a/apps/js-sdk/firecrawl/tsup.config.ts+++ b/apps/js-sdk/firecrawl/tsup.config.ts@@ -8,6 +8,7 @@ export default defineConfig({clean: true,platform: "node",target: "node22",+ external: ["ws"],noExternal: ["typescript-event-target"],esbuildOptions(options) {options.define = {