Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
[sdk] fixes missing headers param in scrape_url (#1795)
Rafael Millerad967d4dbba6
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 370628f36..60a9ea1f3 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -13,7 +13,7 @@ import osfrom .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa-__version__ = "2.15.0"+__version__ = "2.16.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex 062ed3302..4d730a97c 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -464,6 +464,7 @@ class FirecrawlApp:url: str,*,formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None,+ headers: Optional[Dict[str, str]] = None,include_tags: Optional[List[str]] = None,exclude_tags: Optional[List[str]] = None,only_main_content: Optional[bool] = None,@@ -490,6 +491,7 @@ class FirecrawlApp:Args:url (str): Target URL to scrapeformats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc)+ headers (Optional[Dict[str, str]]): Custom HTTP headersinclude_tags (Optional[List[str]]): HTML tags to includeexclude_tags (Optional[List[str]]): HTML tags to excludeonly_main_content (Optional[bool]): Extract main content only@@ -518,7 +520,7 @@ class FirecrawlApp:Raises:Exception: If scraping fails"""- headers = self._prepare_headers()+ _headers = self._prepare_headers()# Build scrape parametersscrape_params = {@@ -529,6 +531,8 @@ class FirecrawlApp:# Add optional parameters if providedif formats:scrape_params['formats'] = formats+ if headers:+ scrape_params['headers'] = headersif include_tags:scrape_params['includeTags'] = include_tagsif exclude_tags:@@ -584,7 +588,7 @@ class FirecrawlApp:# Make requestresponse = requests.post(f'{self.api_url}/v1/scrape',- headers=headers,+ headers=_headers,json=scrape_params,timeout=(timeout + 5000 if timeout else None))@@ -2963,6 +2967,7 @@ class AsyncFirecrawlApp(FirecrawlApp):url: str,*,formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None,+ headers: Optional[Dict[str, str]] = None,include_tags: Optional[List[str]] = None,exclude_tags: Optional[List[str]] = None,only_main_content: Optional[bool] = None,@@ -2985,6 +2990,7 @@ class AsyncFirecrawlApp(FirecrawlApp):Args:url (str): Target URL to scrapeformats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc)+ headers (Optional[Dict[str, str]]): Custom HTTP headersinclude_tags (Optional[List[str]]): HTML tags to includeexclude_tags (Optional[List[str]]): HTML tags to excludeonly_main_content (Optional[bool]): Extract main content only@@ -3019,7 +3025,7 @@ class AsyncFirecrawlApp(FirecrawlApp):# Validate any additional kwargsself._validate_kwargs(kwargs, "scrape_url")- headers = self._prepare_headers()+ _headers = self._prepare_headers()# Build scrape parametersscrape_params = {@@ -3030,6 +3036,8 @@ class AsyncFirecrawlApp(FirecrawlApp):# Add optional parameters if provided and not Noneif formats:scrape_params['formats'] = formats+ if headers:+ scrape_params['headers'] = headersif include_tags:scrape_params['includeTags'] = include_tagsif exclude_tags:@@ -3077,7 +3085,7 @@ class AsyncFirecrawlApp(FirecrawlApp):response = await self._async_post_request(f'{self.api_url}{endpoint}',scrape_params,- headers+ _headers)if response.get('success') and 'data' in response: