🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(Fetcher): Fix the issue of passing cookies
Karim shoair1240ee3eb02e
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 6ef3810..06ee1ee 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -17,6 +17,7 @@ class StaticEngine:follow_redirects: bool = True,timeout: Optional[Union[int, float]] = None,retries: Optional[int] = 3,+ cookies: Optional[Dict] = None,adaptor_arguments: Tuple = None,):"""An engine that utilizes httpx library, check the `Fetcher` class for more documentation.@@ -26,6 +27,7 @@ class StaticEngine:create a referer header as if this request had came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.+ :param cookies: Set cookies for the next request.:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class."""@@ -35,6 +37,7 @@ class StaticEngine:self.timeout = timeoutself.follow_redirects = bool(follow_redirects)self.retries = retries+ self.cookies = dict(cookies) if cookies else {}self._extra_headers = generate_headers(browser_mode=False)# Because we are using `lru_cache` for a slight optimization but both dict/dict_items are not hashable so they can't be cached# So my solution here was to convert it to tuple then convert it back to dictionary again here as tuples are hashable, ofc `tuple().__hash__()`@@ -98,7 +101,9 @@ class StaticEngine:def _make_request(self, method: str, **kwargs) -> Response:headers = self._headers_job(kwargs.pop("headers", {}))with httpx.Client(- proxy=self.proxy, transport=httpx.HTTPTransport(retries=self.retries)+ proxy=self.proxy,+ transport=httpx.HTTPTransport(retries=self.retries),+ cookies=self.cookies,) as client:request = getattr(client, method)(url=self.url,@@ -112,7 +117,9 @@ class StaticEngine:async def _async_make_request(self, method: str, **kwargs) -> Response:headers = self._headers_job(kwargs.pop("headers", {}))async with httpx.AsyncClient(- proxy=self.proxy, transport=httpx.AsyncHTTPTransport(retries=self.retries)+ proxy=self.proxy,+ transport=httpx.AsyncHTTPTransport(retries=self.retries),+ cookies=self.cookies,) as client:request = await getattr(client, method)(url=self.url,diff --git a/scrapling/fetchers.py b/scrapling/fetchers.pyindex 6bb9a6e..1575e20 100644--- a/scrapling/fetchers.py+++ b/scrapling/fetchers.py@@ -31,6 +31,7 @@ class Fetcher(BaseFetcher):stealthy_headers: bool = True,proxy: Optional[str] = None,retries: Optional[int] = 3,+ cookies: Optional[Dict] = None,custom_config: Dict = None,**kwargs: Dict,) -> Response:@@ -43,6 +44,7 @@ class Fetcher(BaseFetcher):create a referer header as if this request had came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.+ :param cookies: Set cookies for the next request.:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.:param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`@@ -57,6 +59,12 @@ class Fetcher(BaseFetcher):adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())++ if not cookies:+ cookies = {}+ elif not isinstance(cookies, dict):+ ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")+response_object = StaticEngine(url,proxy,@@ -64,6 +72,7 @@ class Fetcher(BaseFetcher):follow_redirects,timeout,retries,+ tuple(cookies.items()),adaptor_arguments=adaptor_arguments,).get(**kwargs)return response_object@@ -77,6 +86,7 @@ class Fetcher(BaseFetcher):stealthy_headers: bool = True,proxy: Optional[str] = None,retries: Optional[int] = 3,+ cookies: Optional[Dict] = None,custom_config: Dict = None,**kwargs: Dict,) -> Response:@@ -89,6 +99,7 @@ class Fetcher(BaseFetcher):create a referer header as if this request came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.+ :param cookies: Set cookies for the next request.:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.:param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`@@ -103,6 +114,12 @@ class Fetcher(BaseFetcher):adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())++ if not cookies:+ cookies = {}+ elif not isinstance(cookies, dict):+ ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")+response_object = StaticEngine(url,proxy,@@ -110,6 +127,7 @@ class Fetcher(BaseFetcher):follow_redirects,timeout,retries,+ tuple(cookies.items()),adaptor_arguments=adaptor_arguments,).post(**kwargs)return response_object@@ -123,6 +141,7 @@ class Fetcher(BaseFetcher):stealthy_headers: bool = True,proxy: Optional[str] = None,retries: Optional[int] = 3,+ cookies: Optional[Dict] = None,custom_config: Dict = None,**kwargs: Dict,) -> Response:@@ -135,6 +154,7 @@ class Fetcher(BaseFetcher):create a referer header as if this request came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.+ :param cookies: Set cookies for the next request.:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.:param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.@@ -150,6 +170,12 @@ class Fetcher(BaseFetcher):adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())++ if not cookies:+ cookies = {}+ elif not isinstance(cookies, dict):+ ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")+response_object = StaticEngine(url,proxy,@@ -157,6 +183,7 @@ class Fetcher(BaseFetcher):follow_redirects,timeout,retries,+ tuple(cookies.items()),adaptor_arguments=adaptor_arguments,).put(**kwargs)return response_object@@ -170,6 +197,7 @@ class Fetcher(BaseFetcher):stealthy_headers: bool = True,proxy: Optional[str] = None,retries: Optional[int] = 3,+ cookies: Optional[Dict] = None,custom_config: Dict = None,**kwargs: Dict,) -> Response:@@ -182,6 +210,7 @@ class Fetcher(BaseFetcher):create a referer header as if this request came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.+ :param cookies: Set cookies for the next request.:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.:param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`@@ -196,6 +225,12 @@ class Fetcher(BaseFetcher):adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())++ if not cookies:+ cookies = {}+ elif not isinstance(cookies, dict):+ ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")+response_object = StaticEngine(url,proxy,@@ -203,6 +238,7 @@ class Fetcher(BaseFetcher):follow_redirects,timeout,retries,+ tuple(cookies.items()),adaptor_arguments=adaptor_arguments,).delete(**kwargs)return response_object@@ -218,6 +254,7 @@ class AsyncFetcher(Fetcher):stealthy_headers: bool = True,proxy: Optional[str] = None,retries: Optional[int] = 3,+ cookies: Optional[Dict] = None,custom_config: Dict = None,**kwargs: Dict,) -> Response:@@ -230,6 +267,7 @@ class AsyncFetcher(Fetcher):create a referer header as if this request had came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.+ :param cookies: Set cookies for the next request.:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.:param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`@@ -244,6 +282,12 @@ class AsyncFetcher(Fetcher):adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())++ if not cookies:+ cookies = {}+ elif not isinstance(cookies, dict):+ ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")+response_object = await StaticEngine(url,proxy,@@ -251,6 +295,7 @@ class AsyncFetcher(Fetcher):follow_redirects,timeout,retries=retries,+ cookies=tuple(cookies.items()),adaptor_arguments=adaptor_arguments,).async_get(**kwargs)return response_object@@ -264,6 +309,7 @@ class AsyncFetcher(Fetcher):stealthy_headers: bool = True,proxy: Optional[str] = None,retries: Optional[int] = 3,+ cookies: Optional[Dict] = None,custom_config: Dict = None,**kwargs: Dict,) -> Response:@@ -276,6 +322,7 @@ class AsyncFetcher(Fetcher):create a referer header as if this request came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.+ :param cookies: Set cookies for the next request.:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.:param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`@@ -290,6 +337,12 @@ class AsyncFetcher(Fetcher):adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())++ if not cookies:+ cookies = {}+ elif not isinstance(cookies, dict):+ ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")+response_object = await StaticEngine(url,proxy,@@ -297,6 +350,7 @@ class AsyncFetcher(Fetcher):follow_redirects,timeout,retries=retries,+ cookies=tuple(cookies.items()),adaptor_arguments=adaptor_arguments,).async_post(**kwargs)return response_object@@ -310,6 +364,7 @@ class AsyncFetcher(Fetcher):stealthy_headers: bool = True,proxy: Optional[str] = None,retries: Optional[int] = 3,+ cookies: Optional[Dict] = None,custom_config: Dict = None,**kwargs: Dict,) -> Response:@@ -322,6 +377,7 @@ class AsyncFetcher(Fetcher):create a referer header as if this request came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.+ :param cookies: Set cookies for the next request.:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.:param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`@@ -336,6 +392,12 @@ class AsyncFetcher(Fetcher):adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())++ if not cookies:+ cookies = {}+ elif not isinstance(cookies, dict):+ ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")+response_object = await StaticEngine(url,proxy,@@ -343,6 +405,7 @@ class AsyncFetcher(Fetcher):follow_redirects,timeout,retries=retries,+ cookies=tuple(cookies.items()),adaptor_arguments=adaptor_arguments,).async_put(**kwargs)return response_object@@ -356,6 +419,7 @@ class AsyncFetcher(Fetcher):stealthy_headers: bool = True,proxy: Optional[str] = None,retries: Optional[int] = 3,+ cookies: Optional[Dict] = None,custom_config: Dict = None,**kwargs: Dict,) -> Response:@@ -368,6 +432,7 @@ class AsyncFetcher(Fetcher):create a referer header as if this request came from Google's search of this URL's domain.:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.+ :param cookies: Set cookies for the next request.:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.:param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`@@ -382,6 +447,12 @@ class AsyncFetcher(Fetcher):adaptor_arguments = tuple({**cls._generate_parser_arguments(), **custom_config}.items())++ if not cookies:+ cookies = {}+ elif not isinstance(cookies, dict):+ ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")+response_object = await StaticEngine(url,proxy,@@ -389,6 +460,7 @@ class AsyncFetcher(Fetcher):follow_redirects,timeout,retries=retries,+ cookies=tuple(cookies.items()),adaptor_arguments=adaptor_arguments,).async_delete(**kwargs)return response_object