Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolasb66e0ccb7db4
diff --git a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_full_withAuth/index.test.tsindex 40686c45d..45b29d2b7 100644--- a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts+++ b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts@@ -62,15 +62,6 @@ describe("E2E Tests for API Routes", () => {expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);});- // tested on rate limit test- // it.concurrent("should return a successful response with a valid preview token", async () => {- // const response = await request(TEST_URL)- // .post("/v0/scrape")- // .set("Authorization", `Bearer this_is_just_a_preview_token`)- // .set("Content-Type", "application/json")- // .send({ url: "https://roastmywebsite.ai" });- // expect(response.statusCode).toBe(200);- // }, 30000); // 30 seconds timeoutit.concurrent("should return a successful response with a valid API key",@@ -1087,19 +1078,6 @@ describe("E2E Tests for API Routes", () => {},3000,);-- // it.concurrent("should return a successful response with a valid API key for crawlWebsitePreview", async () => {- // const response = await request(TEST_URL)- // .post("/v0/crawlWebsitePreview")- // .set("Authorization", `Bearer this_is_just_a_preview_token`)- // .set("Content-Type", "application/json")- // .send({ url: "https://firecrawl.dev" });- // expect(response.statusCode).toBe(200);- // expect(response.body).toHaveProperty("jobId");- // expect(response.body.jobId).toMatch(- // /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/- // );- // });});describe("POST /v0/search", () => {@@ -1724,7 +1702,7 @@ describe("E2E Tests for API Routes", () => {for (let i = 0; i < 5; i++) {const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer this_is_just_a_preview_token`)+ .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`).set("Content-Type", "application/json").send({ url: "https://www.scrapethissite.com" });@@ -1732,7 +1710,7 @@ describe("E2E Tests for API Routes", () => {}const response = await request(TEST_URL).post("/v0/scrape")- .set("Authorization", `Bearer this_is_just_a_preview_token`)+ .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`).set("Content-Type", "application/json").send({ url: "https://www.scrapethissite.com" });diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.tsindex 7d107afe6..02bd78471 100644--- a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts@@ -55,7 +55,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {"should return successful response with valid preview token",async () => {const app = new FirecrawlApp<"v0">({- apiKey: "this_is_just_a_preview_token",+ apiKey: process.env.PREVIEW_TOKEN,apiUrl: API_URL,version: "v0",});diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.tsindex 2e601dc4c..d689b0143 100644--- a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts@@ -40,7 +40,7 @@ describe('FirecrawlApp E2E Tests', () => {});test.concurrent('should return successful response with valid preview token', async () => {- const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL });+ const app = new FirecrawlApp({ apiKey: process.env.PREVIEW_TOKEN, apiUrl: API_URL });const response = await app.scrapeUrl('https://roastmywebsite.ai');if (!response.success) {throw new Error(response.error);@@ -365,7 +365,7 @@ describe('FirecrawlApp E2E Tests', () => {});test.concurrent('should return successful response with valid preview token', async () => {- const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL });+ const app = new FirecrawlApp({ apiKey: process.env.PREVIEW_TOKEN, apiUrl: API_URL });const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;expect(response).not.toBeNull();expect(response.links?.length).toBeGreaterThan(0);diff --git a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.pyindex 50d5306f8..bbe5df68e 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py@@ -37,7 +37,7 @@ def test_scrape_url_invalid_api_key():# assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)def test_successful_response_with_valid_preview_token():- app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token", version='v0')+ app = FirecrawlApp(api_url=API_URL, api_key=os.getenv('PREVIEW_TOKEN'), version='v0')response = app.scrape_url('https://roastmywebsite.ai')assert response is not Noneassert 'content' in responsediff --git a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.pyindex eacec8da6..d99ee820c 100644--- a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py+++ b/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py@@ -49,7 +49,7 @@ def test_scrape_url_invalid_api_key():# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)def test_successful_response_with_valid_preview_token():- app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token")+ app = FirecrawlApp(api_url=API_URL, api_key=os.getenv('PREVIEW_TOKEN'))response = app.scrape_url('https://roastmywebsite.ai')assert response is not Noneassert "_Roast_" in response['markdown']@@ -327,7 +327,7 @@ def test_invalid_api_key_on_map():# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)def test_successful_response_with_valid_preview_token_on_map():- app = FirecrawlApp(api_key="this_is_just_a_preview_token", api_url=API_URL)+ app = FirecrawlApp(api_key=os.getenv('PREVIEW_TOKEN'), api_url=API_URL)response = app.map_url('https://roastmywebsite.ai')assert response is not Noneassert len(response) > 0diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rsindex 00f3e26c0..882a29413 100644--- a/apps/rust-sdk/tests/e2e_with_auth.rs+++ b/apps/rust-sdk/tests/e2e_with_auth.rs@@ -26,7 +26,7 @@ async fn test_successful_response_with_valid_preview_token() {let api_url = env::var("API_URL").unwrap();let app = FirecrawlApp::new_selfhosted(api_url,- Some("this_is_just_a_preview_token"),+ Some(env::var("PREVIEW_TOKEN").unwrap()),).unwrap();let result = app