Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge pull request #990 from RutamBhagat/python-sdk-conditionally-enforce-api-key
Nicolascd08be7f378a
diff --cc apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.pyindex 0ada6c1d8,1ed539685..d25d43f33--- a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py+++ b/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py@@@ -20,22 -20,33 +20,33 @@@ spec.loader.exec_module(firecrawlFirecrawlApp = firecrawl.FirecrawlAppdef test_no_api_key():- with pytest.raises(Exception) as excinfo:- invalid_app = FirecrawlApp(api_url=API_URL)- assert "No API key provided" in str(excinfo.value)+ if 'api.firecrawl.dev' in API_URL:+ with pytest.raises(Exception) as excinfo:+ invalid_app = FirecrawlApp(api_url=API_URL)+ assert "No API key provided" in str(excinfo.value)+ else:+ # Should not raise error for self-hosted+ app = FirecrawlApp(api_url=API_URL)+ assert app is not Nonedef test_scrape_url_invalid_api_key():- invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")- with pytest.raises(Exception) as excinfo:- invalid_app.scrape_url('https://firecrawl.dev')- assert "Unauthorized: Invalid token" in str(excinfo.value)+ if 'api.firecrawl.dev' in API_URL:+ invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")+ with pytest.raises(Exception) as excinfo:+ invalid_app.scrape_url('https://firecrawl.dev')+ assert "Unauthorized: Invalid token" in str(excinfo.value)+ else:+ # Should work without API key for self-hosted+ app = FirecrawlApp(api_url=API_URL)+ response = app.scrape_url('https://firecrawl.dev')+ assert response is not None-def test_blocklisted_url():- blocklisted_url = "https://facebook.com/fake-test"- app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)- with pytest.raises(Exception) as excinfo:- app.scrape_url(blocklisted_url)- assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)+# def test_blocklisted_url():+# blocklisted_url = "https://facebook.com/fake-test"+# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)+# with pytest.raises(Exception) as excinfo:+# app.scrape_url(blocklisted_url)+# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)def test_successful_response_with_valid_preview_token():app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token")@@@ -131,17 -142,23 +142,23 @@@ def test_successful_response_for_valid_assert 'We present spectrophotometric observations of the Broad Line Radio Galaxy' in response['markdown']def test_crawl_url_invalid_api_key():- invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")- with pytest.raises(Exception) as excinfo:- invalid_app.crawl_url('https://firecrawl.dev')- assert "Unauthorized: Invalid token" in str(excinfo.value)+ if 'api.firecrawl.dev' in API_URL:+ invalid_app = FirecrawlApp(api_url=API_URL, api_key="invalid_api_key")+ with pytest.raises(Exception) as excinfo:+ invalid_app.crawl_url('https://firecrawl.dev')+ assert "Unauthorized: Invalid token" in str(excinfo.value)+ else:+ # Should work without API key for self-hosted+ app = FirecrawlApp(api_url=API_URL)+ response = app.crawl_url('https://firecrawl.dev')+ assert response is not None-def test_should_return_error_for_blocklisted_url():- app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)- blocklisted_url = "https://twitter.com/fake-test"- with pytest.raises(Exception) as excinfo:- app.crawl_url(blocklisted_url)- assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)+# def test_should_return_error_for_blocklisted_url():+# app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)+# blocklisted_url = "https://twitter.com/fake-test"+# with pytest.raises(Exception) as excinfo:+# app.crawl_url(blocklisted_url)+# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)def test_crawl_url_wait_for_completion_e2e():app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)@@@ -291,17 -308,23 +308,23 @@@ def test_check_crawl_status_e2e()assert 'error' not in status_response['data'][0]['metadata']def test_invalid_api_key_on_map():- invalid_app = FirecrawlApp(api_key="invalid_api_key", api_url=API_URL)- with pytest.raises(Exception) as excinfo:- invalid_app.map_url('https://roastmywebsite.ai')- assert "Unauthorized: Invalid token" in str(excinfo.value)+ if 'api.firecrawl.dev' in API_URL:+ invalid_app = FirecrawlApp(api_key="invalid_api_key", api_url=API_URL)+ with pytest.raises(Exception) as excinfo:+ invalid_app.map_url('https://roastmywebsite.ai')+ assert "Unauthorized: Invalid token" in str(excinfo.value)+ else:+ # Should work without API key for self-hosted+ app = FirecrawlApp(api_url=API_URL)+ response = app.map_url('https://roastmywebsite.ai')+ assert response is not None-def test_blocklisted_url_on_map():- app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)- blocklisted_url = "https://facebook.com/fake-test"- with pytest.raises(Exception) as excinfo:- app.map_url(blocklisted_url)- assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)+# def test_blocklisted_url_on_map():+# app = FirecrawlApp(api_key=TEST_API_KEY, api_url=API_URL)+# blocklisted_url = "https://facebook.com/fake-test"+# with pytest.raises(Exception) as excinfo:+# app.map_url(blocklisted_url)+# assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)def test_successful_response_with_valid_preview_token_on_map():app = FirecrawlApp(api_key="this_is_just_a_preview_token", api_url=API_URL)