🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(requests): Fix multiple configuration issues
Karim shoair9cae97b17708
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 6833982..a775b5b 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -94,7 +94,8 @@ class _ConfigurationLogic(ABC):def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]:"""Merge request-specific arguments with default session arguments."""url = method_kwargs.pop("url")- impersonate = self._get_with_precedence(method_kwargs.get("impersonate"), self._default_impersonate)+ impersonate = self._get_with_precedence(method_kwargs.pop("impersonate"), self._default_impersonate)+ http3_enabled = self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3)final_args = {"url": url,# Curl automatically generates the suitable browser headers when you use `impersonate`@@ -125,7 +126,7 @@ class _ConfigurationLogic(ABC):)}, # Add any remaining parameters (after all known ones are popped)}- if self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3): # pragma: no cover+ if http3_enabled: # pragma: no coverfinal_args["http_version"] = CurlHttpVersion.V3ONLYif impersonate:log.warning(@@ -141,7 +142,7 @@ class _ConfigurationLogic(ABC):3. Generates a referer header that looks like as if this request came from a Google's search of the current URL's domain."""# Merge session headers with request headers, request takes precedence (if it was set)- final_headers = {**self._default_headers, **(headers if headers is not _UNSET else {})}+ final_headers = {**self._default_headers, **(headers if headers and headers is not _UNSET else {})}headers_keys = {k.lower() for k in final_headers}if stealth:if "referer" not in headers_keys:@@ -252,8 +253,6 @@ class _SyncSessionLogic(_ConfigurationLogic):try:response = session.request(method, **request_args)result = ResponseFactory.from_http_request(response, selector_config)- if one_off_request:- session.close()return resultexcept CurlError as e: # pragma: no coverif attempt < max_retries - 1:@@ -667,8 +666,6 @@ class _ASyncSessionLogic(_ConfigurationLogic):try:response = await session.request(method, **request_args)result = ResponseFactory.from_http_request(response, selector_config)- if one_off_request:- await session.close()return resultexcept CurlError as e: # pragma: no coverif attempt < max_retries - 1: