Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Merge pull request #997 from mendableai/feat/sdk-without-ws"
Nicolasb1a5625b2208
diff --git a/apps/js-sdk/firecrawl/src/__tests__/CrawlWatcher.test.ts b/apps/js-sdk/firecrawl/src/__tests__/CrawlWatcher.test.tsdeleted file mode 100644index 7f53828dc..000000000--- a/apps/js-sdk/firecrawl/src/__tests__/CrawlWatcher.test.ts+++ /dev/null@@ -1,35 +0,0 @@-import { jest } from '@jest/globals';--describe('CrawlWatcher', () => {- const mockApiUrl = 'https://api.firecrawl.dev';- const mockApiKey = 'test-api-key';-- beforeEach(() => {- jest.resetModules();- });-- test('should create a CrawlWatcher instance successfully when isows is available', async () => {- await jest.unstable_mockModule('isows', () => ({- WebSocket: jest.fn(),- }));-- const { default: FirecrawlApp, CrawlWatcher } = await import('../index');- const app = new FirecrawlApp({ apiKey: mockApiKey, apiUrl: mockApiUrl });-- const watcher = new CrawlWatcher('test-id', app);- expect(watcher).toBeInstanceOf(CrawlWatcher);- });-- test('should throw when WebSocket is not available (isows import fails)', async () => {- await jest.unstable_mockModule('isows', () => {- throw new Error('Module not found');- });-- const { default: FirecrawlApp, CrawlWatcher, FirecrawlError } = await import('../index');- const app = new FirecrawlApp({ apiKey: mockApiKey, apiUrl: mockApiUrl });-- expect(() => {- new CrawlWatcher('test-id', app);- }).toThrow(FirecrawlError);- });-});diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.tsindex 6db517753..7d107afe6 100644--- a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts@@ -32,7 +32,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {});await expect(invalidApp.scrapeUrl("https://roastmywebsite.ai")- ).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 401");+ ).rejects.toThrow("Request failed with status code 401");});@@ -46,7 +46,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {});const blocklistedUrl = "https://facebook.com/fake-test";await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow(- "Unexpected error occurred while trying to scrape URL. Status code: 403"+ "Request failed with status code 403");});@@ -169,7 +169,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {});const blocklistedUrl = "https://twitter.com/fake-test";await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow(- "Unexpected error occurred while trying to scrape URL. Status code: 403"+ "Request failed with status code 403");});@@ -242,7 +242,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {const maxChecks = 15;let checks = 0;- while ((statusResponse.status === "active" || statusResponse.status === "scraping" ) && checks < maxChecks) {+ while (statusResponse.status === "active" && checks < maxChecks) {await new Promise((resolve) => setTimeout(resolve, 5000));expect(statusResponse.partial_data).not.toBeNull();// expect(statusResponse.current).toBeGreaterThanOrEqual(1);diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.tsindex 4f3a9cb23..e5c042096 100644--- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts@@ -36,7 +36,7 @@ describe('FirecrawlApp E2E Tests', () => {test.concurrent('should throw error for blocklisted URL on scrape', async () => {const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });const blocklistedUrl = "https://facebook.com/fake-test";- await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 403");+ await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403");});test.concurrent('should return successful response with valid preview token', async () => {@@ -74,7 +74,7 @@ describe('FirecrawlApp E2E Tests', () => {'https://roastmywebsite.ai', {formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],headers: { "x-key": "test" },- // includeTags: ['h1'],+ includeTags: ['h1'],excludeTags: ['h2'],onlyMainContent: true,timeout: 30000,@@ -224,7 +224,7 @@ describe('FirecrawlApp E2E Tests', () => {scrapeOptions: {formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],headers: { "x-key": "test" },- // includeTags: ['h1'],+ includeTags: ['h1'],excludeTags: ['h2'],onlyMainContent: true,waitFor: 1000@@ -346,7 +346,7 @@ describe('FirecrawlApp E2E Tests', () => {expect(statusResponse.data[0].metadata).not.toHaveProperty("error");}}- }, 120000); // 120 seconds timeout+ }, 60000); // 60 seconds timeouttest.concurrent('should throw error for invalid API key on map', async () => {if (API_URL.includes('api.firecrawl.dev')) {diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex feb69f03e..d3ae630b8 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -1,23 +1,7 @@import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from "axios";import type * as zt from "zod";import { zodToJsonSchema } from "zod-to-json-schema";--import type { WebSocket as IsowsWebSocket } from 'isows';-/**- * Dynamically imports the WebSocket class from 'isows'.- * If the import fails, WebSocket is set to null.- * This approach is used because some environments, such as Firebase Functions,- * might not support WebSocket natively.- */-const WebSocket: typeof IsowsWebSocket | null = await (async () => {- try {- const module = await import('isows');- return module.WebSocket;- } catch (error) {- return null;- }-})();-+import { WebSocket } from "isows";import { TypedEventTarget } from "typescript-event-target";/**@@ -961,8 +945,6 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {constructor(id: string, app: FirecrawlApp) {super();- if(!WebSocket)- throw new FirecrawlError("WebSocket module failed to load. Your system might not support WebSocket.", 500);this.id = id;this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);this.status = "scraping";