Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(js-sdk): improve API key handling for cloud vs self-hosted services in FirecrawlApp
RutamBhagate776847c71a3
diff --git a/apps/js-sdk/firecrawl/src/__tests__/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/index.test.tsindex 929512374..6958abf80 100644--- a/apps/js-sdk/firecrawl/src/__tests__/index.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/index.test.ts@@ -1,9 +1,9 @@-import { describe, test, expect, jest } from '@jest/globals';-import axios from 'axios';-import FirecrawlApp from '../index';+import { describe, expect, jest, test } from '@jest/globals';-import { readFile } from 'fs/promises';+import FirecrawlApp from '../index';+import axios from 'axios';import { join } from 'path';+import { readFile } from 'fs/promises';// Mock jest and set the typejest.mock('axios');@@ -14,13 +14,22 @@ async function loadFixture(name: string): Promise<string> {return await readFile(join(__dirname, 'fixtures', `${name}.json`), 'utf-8')}+const API_URL = process.env.API_URL ?? "https://api.firecrawl.dev";+describe('the firecrawl JS SDK', () => {- test('Should require an API key to instantiate FirecrawlApp', async () => {- const fn = () => {- new FirecrawlApp({ apiKey: undefined });- };- expect(fn).toThrow('No API key provided');+ test('Should require an API key only for cloud service', async () => {+ if (API_URL.includes('api.firecrawl.dev')) {+ // Should throw for cloud service+ expect(() => {+ new FirecrawlApp({ apiKey: undefined, apiUrl: API_URL });+ }).toThrow('No API key provided');+ } else {+ // Should not throw for self-hosted+ expect(() => {+ new FirecrawlApp({ apiKey: undefined, apiUrl: API_URL });+ }).not.toThrow();+ }});test('Should return scraped data from a /scrape API call', async () => {diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.tsindex dea558462..60d0b44f9 100644--- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts@@ -9,15 +9,28 @@ const TEST_API_KEY = process.env.TEST_API_KEY;const API_URL = process.env.API_URL ?? "https://api.firecrawl.dev";describe('FirecrawlApp E2E Tests', () => {- test.concurrent('should throw error for no API key', async () => {- expect(() => {- new FirecrawlApp({ apiKey: null, apiUrl: API_URL });- }).toThrow("No API key provided");+ test.concurrent('should throw error for no API key only for cloud service', async () => {+ if (API_URL.includes('api.firecrawl.dev')) {+ // Should throw for cloud service+ expect(() => {+ new FirecrawlApp({ apiKey: null, apiUrl: API_URL });+ }).toThrow("No API key provided");+ } else {+ // Should not throw for self-hosted+ expect(() => {+ new FirecrawlApp({ apiKey: null, apiUrl: API_URL });+ }).not.toThrow();+ }});test.concurrent('should throw error for invalid API key on scrape', async () => {- const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });- await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");+ if (API_URL.includes('api.firecrawl.dev')) {+ const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });+ await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 404");+ } else {+ const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });+ await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).resolves.not.toThrow();+ }});test.concurrent('should throw error for blocklisted URL on scrape', async () => {@@ -155,8 +168,13 @@ describe('FirecrawlApp E2E Tests', () => {}, 30000); // 30 seconds timeouttest.concurrent('should throw error for invalid API key on crawl', async () => {- const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });- await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");+ if (API_URL.includes('api.firecrawl.dev')) {+ const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });+ await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 404");+ } else {+ const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });+ await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).resolves.not.toThrow();+ }});test.concurrent('should throw error for blocklisted URL on crawl', async () => {@@ -337,8 +355,13 @@ describe('FirecrawlApp E2E Tests', () => {}, 60000); // 60 seconds timeouttest.concurrent('should throw error for invalid API key on map', async () => {- const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });- await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");+ if (API_URL.includes('api.firecrawl.dev')) {+ const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });+ await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 404");+ } else {+ const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });+ await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).resolves.not.toThrow();+ }});test.concurrent('should throw error for blocklisted URL on map', async () => {@@ -355,8 +378,7 @@ describe('FirecrawlApp E2E Tests', () => {}, 30000); // 30 seconds timeouttest.concurrent('should return successful response for valid map', async () => {- const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });- const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;+ const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;expect(response).not.toBeNull();expect(response.links?.length).toBeGreaterThan(0);diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 020a22936..6d9a0a731 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -289,17 +289,23 @@ export default class FirecrawlApp {public apiKey: string;public apiUrl: string;+ private isCloudService(url: string): boolean {+ return url.includes('api.firecrawl.dev');+ }+/*** Initializes a new instance of the FirecrawlApp class.* @param config - Configuration options for the FirecrawlApp instance.*/constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {- if (typeof apiKey !== "string") {+ const baseUrl = apiUrl || "https://api.firecrawl.dev";++ if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {throw new FirecrawlError("No API key provided", 401);}- this.apiKey = apiKey;- this.apiUrl = apiUrl || "https://api.firecrawl.dev";+ this.apiKey = apiKey || '';+ this.apiUrl = baseUrl;}/**