Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
bugfix zero_data_retention param and certifi dependency (#1749)
Rafael Miller179737104d2d
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 5a90100db..1306587d6 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -13,7 +13,7 @@ import osfrom .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa-__version__ = "2.13.0"+__version__ = "2.14.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex a9243a086..08043ae04 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -23,6 +23,8 @@ import websocketsimport aiohttpimport asynciofrom pydantic import Field+import ssl+import certifi# Suppress Pydantic warnings about attribute shadowingwarnings.filterwarnings("ignore", message="Field name \"json\" in \"FirecrawlDocument\" shadows an attribute in parent \"BaseModel\"")@@ -571,6 +573,8 @@ class FirecrawlApp:scrape_params['maxAge'] = max_ageif store_in_cache is not None:scrape_params['storeInCache'] = store_in_cache+ if zero_data_retention is not None:+ scrape_params['zeroDataRetention'] = zero_data_retentionscrape_params.update(kwargs)@@ -1372,6 +1376,8 @@ class FirecrawlApp:scrape_params['agent'] = agent.dict(exclude_none=True)if max_concurrency is not None:scrape_params['maxConcurrency'] = max_concurrency+ if zero_data_retention is not None:+ scrape_params['zeroDataRetention'] = zero_data_retention# Add any additional kwargsscrape_params.update(kwargs)@@ -1423,6 +1429,7 @@ class FirecrawlApp:agent: Optional[AgentOptions] = None,max_concurrency: Optional[int] = None,idempotency_key: Optional[str] = None,+ zero_data_retention: Optional[bool] = None,**kwargs) -> BatchScrapeResponse:"""@@ -2762,7 +2769,8 @@ class AsyncFirecrawlApp(FirecrawlApp):aiohttp.ClientError: If the request fails after all retries.Exception: If max retries are exceeded or other errors occur."""- async with aiohttp.ClientSession() as session:+ ssl_context = ssl.create_default_context(cafile=certifi.where())+ async with aiohttp.ClientSession(ssl=ssl_context) as session:for attempt in range(retries):try:async with session.request(diff --git a/apps/python-sdk/requirements.txt b/apps/python-sdk/requirements.txtindex 360d9e764..e2d8502de 100644--- a/apps/python-sdk/requirements.txt+++ b/apps/python-sdk/requirements.txt@@ -4,4 +4,5 @@ python-dotenvwebsocketsnest-asynciopydantic-aiohttp\ No newline at end of file+aiohttp+certifi\ No newline at end of filediff --git a/apps/python-sdk/setup.py b/apps/python-sdk/setup.pyindex 1fb31664e..8bc9c505a 100644--- a/apps/python-sdk/setup.py+++ b/apps/python-sdk/setup.py@@ -34,7 +34,8 @@ setup('asyncio','nest-asyncio','pydantic',- 'aiohttp'+ 'aiohttp',+ 'certifi'],python_requires=">=3.8",classifiers=[