Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Enhance error handling in E2E tests and introduce CrawlWatcher tests
Thomas Kosmasf043f5fd61d2
diff --git a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.tsindex 35ee2d89f..f5fc5d5d0 100644--- a/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts+++ b/apps/api/src/__tests__/e2e_v1_withAuth/index.test.ts@@ -58,7 +58,7 @@ describe("E2E Tests for v1 API Routes", () => {expect(response.statusCode).toBe(403);expect(response.body.error).toBe(- "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.",+ "Request failed with status code 403. Error: URL is blocked intentionally. Firecrawl currently does not support scraping this site due to policy restrictions. ",);});@@ -757,7 +757,7 @@ describe("E2E Tests for v1 API Routes", () => {expect(response.statusCode).toBe(403);expect(response.body.error).toBe(- "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.",+ "Request failed with status code 403. Error: URL is blocked intentionally. Firecrawl currently does not support scraping this site due to policy restrictions. ",);});diff --git a/apps/js-sdk/firecrawl/src/__tests__/CrawlWatcher.test.ts b/apps/js-sdk/firecrawl/src/__tests__/CrawlWatcher.test.tsnew file mode 100644index 000000000..7f53828dc--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/CrawlWatcher.test.ts@@ -0,0 +1,35 @@+import { jest } from '@jest/globals';++describe('CrawlWatcher', () => {+ const mockApiUrl = 'https://api.firecrawl.dev';+ const mockApiKey = 'test-api-key';++ beforeEach(() => {+ jest.resetModules();+ });++ test('should create a CrawlWatcher instance successfully when isows is available', async () => {+ await jest.unstable_mockModule('isows', () => ({+ WebSocket: jest.fn(),+ }));++ const { default: FirecrawlApp, CrawlWatcher } = await import('../index');+ const app = new FirecrawlApp({ apiKey: mockApiKey, apiUrl: mockApiUrl });++ const watcher = new CrawlWatcher('test-id', app);+ expect(watcher).toBeInstanceOf(CrawlWatcher);+ });++ test('should throw when WebSocket is not available (isows import fails)', async () => {+ await jest.unstable_mockModule('isows', () => {+ throw new Error('Module not found');+ });++ const { default: FirecrawlApp, CrawlWatcher, FirecrawlError } = await import('../index');+ const app = new FirecrawlApp({ apiKey: mockApiKey, apiUrl: mockApiUrl });++ expect(() => {+ new CrawlWatcher('test-id', app);+ }).toThrow(FirecrawlError);+ });+});diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.tsindex 7d107afe6..6db517753 100644--- a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts@@ -32,7 +32,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {});await expect(invalidApp.scrapeUrl("https://roastmywebsite.ai")- ).rejects.toThrow("Request failed with status code 401");+ ).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 401");});@@ -46,7 +46,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {});const blocklistedUrl = "https://facebook.com/fake-test";await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow(- "Request failed with status code 403"+ "Unexpected error occurred while trying to scrape URL. Status code: 403");});@@ -169,7 +169,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {});const blocklistedUrl = "https://twitter.com/fake-test";await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow(- "Request failed with status code 403"+ "Unexpected error occurred while trying to scrape URL. Status code: 403");});@@ -242,7 +242,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {const maxChecks = 15;let checks = 0;- while (statusResponse.status === "active" && checks < maxChecks) {+ while ((statusResponse.status === "active" || statusResponse.status === "scraping" ) && checks < maxChecks) {await new Promise((resolve) => setTimeout(resolve, 5000));expect(statusResponse.partial_data).not.toBeNull();// expect(statusResponse.current).toBeGreaterThanOrEqual(1);diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.tsindex dea558462..76dc7f733 100644--- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts@@ -17,13 +17,13 @@ describe('FirecrawlApp E2E Tests', () => {test.concurrent('should throw error for invalid API key on scrape', async () => {const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });- await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");+ await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 401");});test.concurrent('should throw error for blocklisted URL on scrape', async () => {const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });const blocklistedUrl = "https://facebook.com/fake-test";- await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403");+ await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 403");});test.concurrent('should return successful response with valid preview token', async () => {@@ -61,7 +61,7 @@ describe('FirecrawlApp E2E Tests', () => {'https://roastmywebsite.ai', {formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],headers: { "x-key": "test" },- includeTags: ['h1'],+ // includeTags: ['h1'],excludeTags: ['h2'],onlyMainContent: true,timeout: 30000,@@ -162,7 +162,7 @@ describe('FirecrawlApp E2E Tests', () => {test.concurrent('should throw error for blocklisted URL on crawl', async () => {const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });const blocklistedUrl = "https://twitter.com/fake-test";- await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.");+ await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403. Error: This website is no longer supported, please reach out to [email protected] for more info on how to activate it on your account. ");});test.concurrent('should return successful response for crawl and wait for completion', async () => {@@ -212,7 +212,7 @@ describe('FirecrawlApp E2E Tests', () => {scrapeOptions: {formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],headers: { "x-key": "test" },- includeTags: ['h1'],+ // includeTags: ['h1'],excludeTags: ['h2'],onlyMainContent: true,waitFor: 1000@@ -334,7 +334,7 @@ describe('FirecrawlApp E2E Tests', () => {expect(statusResponse.data[0].metadata).not.toHaveProperty("error");}}- }, 60000); // 60 seconds timeout+ }, 120000); // 120 seconds timeouttest.concurrent('should throw error for invalid API key on map', async () => {const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });