Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor: Making the requests logic better and cleaner

Karim shoaire53ce3f74975
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 1c2dbbc..448f0de 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -13,7 +13,6 @@ from scrapling.core._types import (
Callable,
Iterable,
SelectorWaitStates,
- cast,
overload,
)
from scrapling.engines.toolbelt.navigation import construct_proxy_dict
@@ -215,7 +214,7 @@ def validate_fetch(params: List[Tuple], sentinel=None) -> _fetch_params:
result[arg] = session_value
if overrides:
- overrides = cast(FetchConfig, validate(overrides, FetchConfig)).to_dict()
+ overrides = validate(overrides, FetchConfig).to_dict()
overrides.update(result)
return _fetch_params(**overrides)
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index d49b5da..6833982 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -1,9 +1,9 @@
+from abc import ABC
from time import sleep as time_sleep
from asyncio import sleep as asyncio_sleep
from curl_cffi.curl import CurlError
from curl_cffi import CurlHttpVersion
-from curl_cffi.requests.impersonate import DEFAULT_CHROME
from curl_cffi.requests import (
ProxySpec,
CookieTypes,
@@ -22,7 +22,6 @@ from scrapling.core._types import (
Awaitable,
List,
Any,
- cast,
)
from .toolbelt.custom import Response
@@ -30,22 +29,14 @@ from .toolbelt.convertor import ResponseFactory
from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__
_UNSET: Any = object()
+_NO_SESSION: Any = object()
-class FetcherSession:
- """
- A context manager that provides configured Fetcher sessions.
-
- When this manager is used in a 'with' or 'async with' block,
- it yields a new session configured with the manager's defaults.
- A single instance of this manager should ideally be used for one active
- session at a time (or sequentially). Re-entering a context with the
- same manager instance while a session is already active is disallowed.
- """
-
+class _ConfigurationLogic(ABC):
+ # Core Logic Handler (Internal Engine)
def __init__(
self,
- impersonate: Optional[BrowserTypeLiteral] = DEFAULT_CHROME,
+ impersonate: Optional[BrowserTypeLiteral] = "chrome",
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
proxies: Optional[Dict[str, str]] = None,
@@ -79,185 +70,191 @@ class FetcherSession:
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param selector_config: Arguments passed when creating the final Selector class.
"""
- self.default_impersonate = impersonate
- self.stealth = stealthy_headers
- self.default_proxies = proxies or {}
- self.default_proxy = proxy or None
- self.default_proxy_auth = proxy_auth or None
- self.default_timeout = timeout
- self.default_headers = headers or {}
- self.default_retries = retries
- self.default_retry_delay = retry_delay
- self.default_follow_redirects = follow_redirects
- self.default_max_redirects = max_redirects
- self.default_verify = verify
- self.default_cert = cert
- self.default_http3 = http3
+ self._default_impersonate = impersonate
+ self._stealth = stealthy_headers
+ self._default_proxies = proxies or {}
+ self._default_proxy = proxy or None
+ self._default_proxy_auth = proxy_auth or None
+ self._default_timeout = timeout
+ self._default_headers = headers or {}
+ self._default_retries = retries
+ self._default_retry_delay = retry_delay
+ self._default_follow_redirects = follow_redirects
+ self._default_max_redirects = max_redirects
+ self._default_verify = verify
+ self._default_cert = cert
+ self._default_http3 = http3
self.selector_config = selector_config or {}
- self._curl_session: Optional[CurlSession] = None
- self._async_curl_session: Optional[AsyncCurlSession] = None
+ @staticmethod
+ def _get_with_precedence(request_val: Any, default_val: Any) -> Any:
+ """Get value with request-level priority over session-level"""
+ return request_val if request_val is not _UNSET else default_val
- def _merge_request_args(self, **kwargs) -> Dict[str, Any]:
+ def _merge_request_args(self, **method_kwargs) -> Dict[str, Any]:
"""Merge request-specific arguments with default session arguments."""
- url = kwargs.pop("url")
- request_args = {}
-
- headers = self.get_with_precedence(kwargs, "headers", self.default_headers)
- stealth = self.get_with_precedence(kwargs, "stealth", self.stealth)
- impersonate = self.get_with_precedence(kwargs, "impersonate", self.default_impersonate)
-
- if self.get_with_precedence(kwargs, "http3", self.default_http3): # pragma: no cover
- request_args["http_version"] = CurlHttpVersion.V3ONLY
+ url = method_kwargs.pop("url")
+ impersonate = self._get_with_precedence(method_kwargs.get("impersonate"), self._default_impersonate)
+ final_args = {
+ "url": url,
+ # Curl automatically generates the suitable browser headers when you use `impersonate`
+ "headers": self._headers_job(
+ url,
+ self._get_with_precedence(method_kwargs.pop("headers"), self._default_headers),
+ self._get_with_precedence(method_kwargs.pop("stealth"), self._stealth),
+ bool(impersonate),
+ ),
+ "proxies": self._get_with_precedence(method_kwargs.pop("proxies"), self._default_proxies),
+ "proxy": self._get_with_precedence(method_kwargs.pop("proxy"), self._default_proxy),
+ "proxy_auth": self._get_with_precedence(method_kwargs.pop("proxy_auth"), self._default_proxy_auth),
+ "timeout": self._get_with_precedence(method_kwargs.pop("timeout"), self._default_timeout),
+ "allow_redirects": self._get_with_precedence(
+ method_kwargs.pop("follow_redirects"), self._default_follow_redirects
+ ),
+ "max_redirects": self._get_with_precedence(method_kwargs.pop("max_redirects"), self._default_max_redirects),
+ "verify": self._get_with_precedence(method_kwargs.pop("verify"), self._default_verify),
+ "cert": self._get_with_precedence(method_kwargs.pop("cert"), self._default_cert),
+ "impersonate": impersonate,
+ **{
+ k: v
+ for k, v in method_kwargs.items()
+ if v
+ not in (
+ _UNSET,
+ None,
+ )
+ }, # Add any remaining parameters (after all known ones are popped)
+ }
+ if self._get_with_precedence(method_kwargs.pop("http3"), self._default_http3): # pragma: no cover
+ final_args["http_version"] = CurlHttpVersion.V3ONLY
if impersonate:
log.warning(
"The argument `http3` might cause errors if used with `impersonate` argument, try switching it off if you encounter any curl errors."
)
- request_args.update(
- {
- "url": url,
- # Curl automatically generates the suitable browser headers when you use `impersonate`
- "headers": self._headers_job(url, headers, stealth, bool(impersonate)),
- "proxies": self.get_with_precedence(kwargs, "proxies", self.default_proxies),
- "proxy": self.get_with_precedence(kwargs, "proxy", self.default_proxy),
- "proxy_auth": self.get_with_precedence(kwargs, "proxy_auth", self.default_proxy_auth),
- "timeout": self.get_with_precedence(kwargs, "timeout", self.default_timeout),
- "allow_redirects": self.get_with_precedence(kwargs, "allow_redirects", self.default_follow_redirects),
- "max_redirects": self.get_with_precedence(kwargs, "max_redirects", self.default_max_redirects),
- "verify": self.get_with_precedence(kwargs, "verify", self.default_verify),
- "cert": self.get_with_precedence(kwargs, "cert", self.default_cert),
- "impersonate": impersonate,
- **{
- k: v
- for k, v in kwargs.items()
- if v
- not in (
- _UNSET,
- None,
- )
- }, # Add any remaining parameters (after all known ones are popped)
- }
- )
- return request_args
-
- def _headers_job(
- self,
- url,
- headers: Optional[Dict],
- stealth: Optional[bool],
- impersonate_enabled: bool,
- ) -> Dict:
- """Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and
- finally generates a referer header that looks like if this request came from Google's search of the current URL's domain.
+ return final_args
- :param headers: Current headers in the request if the user passed any
- :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value.
- :param impersonate_enabled: Whether the browser impersonation is enabled or not.
- :return: A dictionary of the new headers.
+ def _headers_job(self, url, headers: Dict, stealth: bool, impersonate_enabled: bool) -> Dict:
"""
- # Handle headers - if it was _UNSET, use default_headers
- if headers is _UNSET:
- headers = self.default_headers.copy()
- else:
- # Merge session headers with request headers, request takes precedence
- headers = {**self.default_headers, **(headers or {})}
-
- headers_keys = set(map(str.lower, headers.keys()))
+ 1. Adds a useragent to the headers if it doesn't have one
+ 2. Generates real headers and append them to current headers
+ 3. Generates a referer header that looks like as if this request came from a Google's search of the current URL's domain.
+ """
+ # Merge session headers with request headers, request takes precedence (if it was set)
+ final_headers = {**self._default_headers, **(headers if headers is not _UNSET else {})}
+ headers_keys = {k.lower() for k in final_headers}
if stealth:
if "referer" not in headers_keys:
- headers.update({"referer": generate_convincing_referer(url)})
-
- if impersonate_enabled: # Curl will generate the suitable headers
- return headers
+ final_headers["referer"] = generate_convincing_referer(url)
- extra_headers = generate_headers(browser_mode=False)
- # Don't overwrite user-supplied headers
- extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys}
- headers.update(extra_headers)
+ if not impersonate_enabled: # Curl will generate the suitable headers
+ extra_headers = generate_headers(browser_mode=False)
+ final_headers.update(
+ {k: v for k, v in extra_headers.items() if k.lower() not in headers_keys}
+ ) # Don't overwrite user-supplied headers
elif "user-agent" not in headers_keys and not impersonate_enabled:
- headers["User-Agent"] = __default_useragent__
- log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")
+ final_headers["User-Agent"] = __default_useragent__
+ log.debug(f"Can't find useragent in headers so '{final_headers['User-Agent']}' was used.")
+
+ return final_headers
+
- return headers
+class _SyncSessionLogic(_ConfigurationLogic):
+ def __init__(
+ self,
+ impersonate: Optional[BrowserTypeLiteral] = "chrome",
+ http3: Optional[bool] = False,
+ stealthy_headers: Optional[bool] = True,
+ proxies: Optional[Dict[str, str]] = None,
+ proxy: Optional[str] = None,
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ timeout: Optional[int | float] = 30,
+ headers: Optional[Dict[str, str]] = None,
+ retries: Optional[int] = 3,
+ retry_delay: Optional[int] = 1,
+ follow_redirects: bool = True,
+ max_redirects: int = 30,
+ verify: bool = True,
+ cert: Optional[str | Tuple[str, str]] = None,
+ selector_config: Optional[Dict] = None,
+ ):
+ super().__init__(
+ impersonate,
+ http3,
+ stealthy_headers,
+ proxies,
+ proxy,
+ proxy_auth,
+ timeout,
+ headers,
+ retries,
+ retry_delay,
+ follow_redirects,
+ max_redirects,
+ verify,
+ cert,
+ selector_config,
+ )
+ self._curl_session: Optional[CurlSession] = None
- def __enter__(self) -> "FetcherClient":
+ def __enter__(self):
"""Creates and returns a new synchronous Fetcher Session"""
if self._curl_session:
- raise RuntimeError(
- "This FetcherSession instance already has an active synchronous session. "
- "Create a new FetcherSession instance for a new independent session, "
- "or use the current instance sequentially after the previous context has exited."
- )
- if self._async_curl_session: # Prevent mixing if async is active from this instance
- raise RuntimeError(
- "This FetcherSession instance has an active asynchronous session. "
- "Cannot enter a synchronous context simultaneously with the same manager instance."
- )
+ raise RuntimeError("This FetcherSession instance already has an active synchronous session.")
self._curl_session = CurlSession()
- return cast("FetcherClient", self)
+ return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Closes the active synchronous session managed by this instance, if any."""
+ # For type checking (not accessed error)
+ _ = (
+ exc_type,
+ exc_val,
+ exc_tb,
+ )
if self._curl_session:
self._curl_session.close()
self._curl_session = None
- async def __aenter__(self) -> "AsyncFetcherClient":
- """Creates and returns a new asynchronous Session."""
- if self._async_curl_session:
- raise RuntimeError(
- "This FetcherSession instance already has an active asynchronous session. "
- "Create a new FetcherSession instance for a new independent session, "
- "or use the current instance sequentially after the previous context has exited."
- )
- if self._curl_session: # Prevent mixing if sync is active from this instance
- raise RuntimeError(
- "This FetcherSession instance has an active synchronous session. "
- "Cannot enter an asynchronous context simultaneously with the same manager instance."
- )
-
- self._async_curl_session = AsyncCurlSession()
- return cast("AsyncFetcherClient", self)
-
- async def __aexit__(self, exc_type, exc_val, exc_tb):
- """Closes the active asynchronous session managed by this instance, if any."""
- if self._async_curl_session:
- await self._async_curl_session.close()
- self._async_curl_session = None
-
def __make_request(
self,
method: SUPPORTED_HTTP_METHODS,
- request_args: Dict[str, Any],
- max_retries: int,
- retry_delay: int,
- selector_config: Dict,
+ stealth: Optional[bool] = None,
+ **kwargs,
) -> Response:
"""
Perform an HTTP request using the configured session.
:param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
- :param request_args: Arguments to be passed to the session's `request()` method.
- :param max_retries: Maximum number of retries for the request.
- :param retry_delay: Number of seconds to wait between retries.
- :param selector_config: Arguments passed when creating the final Selector class.
- :return: A `Response` object for synchronous requests or an awaitable for asynchronous.
+ :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value.
+ :param kwargs: Additional request-specific arguments.
+ :return: A `Response` object.
"""
+ stealth = self._stealth if stealth is None else stealth
+
+ selector_config = kwargs.pop("selector_config", {}) or self.selector_config
+ max_retries = self._get_with_precedence(kwargs.pop("retries"), self._default_retries)
+ retry_delay = self._get_with_precedence(kwargs.pop("retry_delay"), self._default_retry_delay)
+ request_args = self._merge_request_args(stealth=stealth, **kwargs)
+
session = self._curl_session
- if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)):
+ one_off_request = False
+ if session is _NO_SESSION and self.__enter__ is None:
# For usage inside FetcherClient
# It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time.
session = CurlSession()
+ one_off_request = True
if session:
for attempt in range(max_retries):
try:
response = session.request(method, **request_args)
- # response.raise_for_status() # Retry responses with a status code between 200-400
- return ResponseFactory.from_http_request(response, selector_config)
+ result = ResponseFactory.from_http_request(response, selector_config)
+ if one_off_request:
+ session.close()
+ return result
except CurlError as e: # pragma: no cover
if attempt < max_retries - 1:
log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...")
@@ -265,86 +262,12 @@ class FetcherSession:
else:
log.error(f"Failed after {max_retries} attempts: {e}")
raise # Raise the exception if all retries fail
+ finally:
+ if session and one_off_request:
+ session.close()
raise RuntimeError("No active session available.") # pragma: no cover
- async def __make_async_request(
- self,
- method: SUPPORTED_HTTP_METHODS,
- request_args: Dict[str, Any],
- max_retries: int,
- retry_delay: int,
- selector_config: Dict,
- ) -> Response:
- """
- Perform an HTTP request using the configured session.
-
- :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
- :param request_args: Arguments to be passed to the session's `request()` method.
- :param max_retries: Maximum number of retries for the request.
- :param retry_delay: Number of seconds to wait between retries.
- :param selector_config: Arguments passed when creating the final Selector class.
- :return: A `Response` object for synchronous requests or an awaitable for asynchronous.
- """
- session = self._async_curl_session
- if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)):
- # For usage inside the ` AsyncFetcherClient ` class, and that's for several reasons
- # 1. It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time.
- # 2. `curl_cffi` doesn't support making async requests without sessions
- # 3. Using a single session for many requests at the same time in async doesn't sit well with curl_cffi.
- session = AsyncCurlSession()
-
- if session:
- for attempt in range(max_retries):
- try:
- response = await session.request(method, **request_args)
- # response.raise_for_status() # Retry responses with a status code between 200-400
- return ResponseFactory.from_http_request(response, selector_config)
- except CurlError as e: # pragma: no cover
- if attempt < max_retries - 1:
- log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...")
- await asyncio_sleep(retry_delay)
- else:
- log.error(f"Failed after {max_retries} attempts: {e}")
- raise # Raise the exception if all retries fail
-
- raise RuntimeError("No active session available.") # pragma: no cover
-
- @staticmethod
- def get_with_precedence(kwargs, key, default_value):
- """Get value with request-level priority over session-level"""
- request_value = kwargs.pop(key, _UNSET)
- return request_value if request_value is not _UNSET else default_value
-
- def __prepare_and_dispatch(
- self,
- method: SUPPORTED_HTTP_METHODS,
- stealth: Optional[bool] = None,
- **kwargs,
- ) -> Response | Awaitable[Response]:
- """
- Internal dispatcher. Prepares arguments and calls sync or async request helper.
-
- :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
- :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value.
- :param url: Target URL for the request.
- :param kwargs: Additional request-specific arguments.
- :return: A `Response` object for synchronous requests or an awaitable for asynchronous.
- """
- stealth = self.stealth if stealth is None else stealth
-
- selector_config = kwargs.pop("selector_config", {}) or self.selector_config
- max_retries = self.get_with_precedence(kwargs, "retries", self.default_retries)
- retry_delay = self.get_with_precedence(kwargs, "retry_delay", self.default_retry_delay)
- request_args = self._merge_request_args(stealth=stealth, **kwargs)
- if self._curl_session:
- return self.__make_request(method, request_args, max_retries, retry_delay, selector_config)
- elif self._async_curl_session:
- # The returned value is a Coroutine
- return self.__make_async_request(method, request_args, max_retries, retry_delay, selector_config)
-
- raise RuntimeError("No active session available.")
-
def get(
self,
url: str,
@@ -366,7 +289,7 @@ class FetcherSession:
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
- ) -> Response | Awaitable[Response]:
+ ) -> Response:
"""
Perform a GET request.
@@ -390,29 +313,31 @@ class FetcherSession:
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
- :return: A `Response` object or an awaitable for async.
+ :return: A `Response` object.
"""
- request_args = {
- "url": url,
- "params": params,
- "headers": headers,
- "cookies": cookies,
- "timeout": timeout,
- "retry_delay": retry_delay,
- "allow_redirects": follow_redirects,
- "max_redirects": max_redirects,
- "retries": retries,
- "proxies": proxies,
- "proxy": proxy,
- "proxy_auth": proxy_auth,
- "auth": auth,
- "verify": verify,
- "cert": cert,
- "impersonate": impersonate,
- "http3": http3,
- **kwargs,
- }
- return self.__prepare_and_dispatch("GET", stealth=stealthy_headers, **request_args)
+ method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")}
+ method_args.update(kwargs)
+ # For type checking (not accessed error)
+ _ = (
+ url,
+ params,
+ headers,
+ cookies,
+ timeout,
+ follow_redirects,
+ max_redirects,
+ retries,
+ retry_delay,
+ proxies,
+ proxy,
+ proxy_auth,
+ auth,
+ verify,
+ cert,
+ impersonate,
+ http3,
+ )
+ return self.__make_request("GET", stealth=stealthy_headers, **method_args)
def post(
self,
@@ -437,57 +362,59 @@ class FetcherSession:
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
- ) -> Response | Awaitable[Response]:
+ ) -> Response:
"""
Perform a POST request.
:param url: Target URL for the request.
:param data: Form data to include in the request body.
:param json: A JSON serializable object to include in the body of the request.
- :param headers: Headers to include in the request.
:param params: Query string parameters for the request.
+ :param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
- :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
+ :param proxies: Dict of proxies to use.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
- :param verify: Whether to verify HTTPS certificates. Defaults to True.
+ :param verify: Whether to verify HTTPS certificates.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
- :return: A `Response` object or an awaitable for async.
+ :return: A `Response` object.
"""
- request_args = {
- "url": url,
- "data": data,
- "json": json,
- "headers": headers,
- "params": params,
- "cookies": cookies,
- "timeout": timeout,
- "retry_delay": retry_delay,
- "proxy": proxy,
- "impersonate": impersonate,
- "allow_redirects": follow_redirects,
- "max_redirects": max_redirects,
- "retries": retries,
- "proxies": proxies,
- "proxy_auth": proxy_auth,
- "auth": auth,
- "verify": verify,
- "cert": cert,
- "http3": http3,
- **kwargs,
- }
- return self.__prepare_and_dispatch("POST", stealth=stealthy_headers, **request_args)
+ method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")}
+ method_args.update(kwargs)
+ # For type checking (not accessed error)
+ _ = (
+ url,
+ params,
+ headers,
+ data,
+ json,
+ cookies,
+ timeout,
+ follow_redirects,
+ max_redirects,
+ retries,
+ retry_delay,
+ proxies,
+ proxy,
+ proxy_auth,
+ auth,
+ verify,
+ cert,
+ impersonate,
+ http3,
+ )
+ return self.__make_request("POST", stealth=stealthy_headers, **method_args)
def put(
self,
@@ -512,57 +439,59 @@ class FetcherSession:
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
- ) -> Response | Awaitable[Response]:
+ ) -> Response:
"""
Perform a PUT request.
:param url: Target URL for the request.
:param data: Form data to include in the request body.
:param json: A JSON serializable object to include in the body of the request.
- :param headers: Headers to include in the request.
:param params: Query string parameters for the request.
+ :param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
- :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
+ :param proxies: Dict of proxies to use.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
- :param verify: Whether to verify HTTPS certificates. Defaults to True.
+ :param verify: Whether to verify HTTPS certificates.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
- :return: A `Response` object or an awaitable for async.
+ :return: A `Response` object.
"""
- request_args = {
- "url": url,
- "data": data,
- "json": json,
- "headers": headers,
- "params": params,
- "cookies": cookies,
- "timeout": timeout,
- "retry_delay": retry_delay,
- "proxy": proxy,
- "impersonate": impersonate,
- "allow_redirects": follow_redirects,
- "max_redirects": max_redirects,
- "retries": retries,
- "proxies": proxies,
- "proxy_auth": proxy_auth,
- "auth": auth,
- "verify": verify,
- "cert": cert,
- "http3": http3,
- **kwargs,
- }
- return self.__prepare_and_dispatch("PUT", stealth=stealthy_headers, **request_args)
+ method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")}
+ method_args.update(kwargs)
+ # For type checking (not accessed error)
+ _ = (
+ url,
+ params,
+ headers,
+ data,
+ json,
+ cookies,
+ timeout,
+ follow_redirects,
+ max_redirects,
+ retries,
+ retry_delay,
+ proxies,
+ proxy,
+ proxy_auth,
+ auth,
+ verify,
+ cert,
+ impersonate,
+ http3,
+ )
+ return self.__make_request("PUT", stealth=stealthy_headers, **method_args)
def delete(
self,
@@ -587,282 +516,173 @@ class FetcherSession:
http3: Optional[bool] = _UNSET,
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
- ) -> Response | Awaitable[Response]:
+ ) -> Response:
"""
Perform a DELETE request.
:param url: Target URL for the request.
:param data: Form data to include in the request body.
:param json: A JSON serializable object to include in the body of the request.
- :param headers: Headers to include in the request.
:param params: Query string parameters for the request.
+ :param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
:param follow_redirects: Whether to follow redirects. Defaults to True.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
- :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
+ :param proxies: Dict of proxies to use.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
:param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
:param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
- :param verify: Whether to verify HTTPS certificates. Defaults to True.
+ :param verify: Whether to verify HTTPS certificates.
:param cert: Tuple of (cert, key) filenames for the client certificate.
:param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
:param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
- :return: A `Response` object or an awaitable for async.
+ :return: A `Response` object.
"""
- request_args = {
- "url": url,
- # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5,
- # But some websites accept it, it depends on the implementation used.
- "data": data,
- "json": json,
- "headers": headers,
- "params": params,
- "cookies": cookies,
- "timeout": timeout,
- "retry_delay": retry_delay,
- "proxy": proxy,
- "impersonate": impersonate,
- "allow_redirects": follow_redirects,
- "max_redirects": max_redirects,
- "retries": retries,
- "proxies": proxies,
- "proxy_auth": proxy_auth,
- "auth": auth,
- "verify": verify,
- "cert": cert,
- "http3": http3,
- **kwargs,
- }
- return self.__prepare_and_dispatch("DELETE", stealth=stealthy_headers, **request_args)
-
+ # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5,
+ # But some websites accept it, it depends on the implementation used.
+ method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")}
+ method_args.update(kwargs)
+ # For type checking (not accessed error)
+ _ = (
+ url,
+ params,
+ headers,
+ data,
+ json,
+ cookies,
+ timeout,
+ follow_redirects,
+ max_redirects,
+ retries,
+ retry_delay,
+ proxies,
+ proxy,
+ proxy_auth,
+ auth,
+ verify,
+ cert,
+ impersonate,
+ http3,
+ )
+ return self.__make_request("DELETE", stealth=stealthy_headers, **method_args)
-class FetcherClient(FetcherSession):
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.__enter__: Any = None
- self.__exit__: Any = None
- self.__aenter__: Any = None
- self.__aexit__: Any = None
- self._curl_session: Any = True
- # Setting the correct return types for the type checking/autocompletion
- def get(
+class _ASyncSessionLogic(_ConfigurationLogic):
+ def __init__(
self,
- url: str,
- params: Optional[Dict | List | Tuple] = None,
- headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
- cookies: Optional[CookieTypes] = None,
- timeout: Optional[int | float] = _UNSET,
- follow_redirects: Optional[bool] = _UNSET,
- max_redirects: Optional[int] = _UNSET,
- retries: Optional[int] = _UNSET,
- retry_delay: Optional[int] = _UNSET,
- proxies: Optional[ProxySpec] = _UNSET,
- proxy: Optional[str] = _UNSET,
- proxy_auth: Optional[Tuple[str, str]] = _UNSET,
- auth: Optional[Tuple[str, str]] = None,
- verify: Optional[bool] = _UNSET,
- cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
- http3: Optional[bool] = _UNSET,
- stealthy_headers: Optional[bool] = _UNSET,
- **kwargs,
- ) -> Response:
- return cast(
- Response,
- super().get(
- url,
- params,
- headers,
- cookies,
- timeout,
- follow_redirects,
- max_redirects,
- retries,
- retry_delay,
- proxies,
- proxy,
- proxy_auth,
- auth,
- verify,
- cert,
- impersonate,
- http3,
- stealthy_headers,
- **kwargs,
- ),
+ impersonate: Optional[BrowserTypeLiteral] = "chrome",
+ http3: Optional[bool] = False,
+ stealthy_headers: Optional[bool] = True,
+ proxies: Optional[Dict[str, str]] = None,
+ proxy: Optional[str] = None,
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ timeout: Optional[int | float] = 30,
+ headers: Optional[Dict[str, str]] = None,
+ retries: Optional[int] = 3,
+ retry_delay: Optional[int] = 1,
+ follow_redirects: bool = True,
+ max_redirects: int = 30,
+ verify: bool = True,
+ cert: Optional[str | Tuple[str, str]] = None,
+ selector_config: Optional[Dict] = None,
+ ):
+ super().__init__(
+ impersonate,
+ http3,
+ stealthy_headers,
+ proxies,
+ proxy,
+ proxy_auth,
+ timeout,
+ headers,
+ retries,
+ retry_delay,
+ follow_redirects,
+ max_redirects,
+ verify,
+ cert,
+ selector_config,
)
+ self._async_curl_session: Optional[AsyncCurlSession] = None
- def post(
- self,
- url: str,
- data: Optional[Dict | str] = None,
- json: Optional[Dict | List] = None,
- headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
- params: Optional[Dict | List | Tuple] = None,
- cookies: Optional[CookieTypes] = None,
- timeout: Optional[int | float] = _UNSET,
- follow_redirects: Optional[bool] = _UNSET,
- max_redirects: Optional[int] = _UNSET,
- retries: Optional[int] = _UNSET,
- retry_delay: Optional[int] = _UNSET,
- proxies: Optional[ProxySpec] = _UNSET,
- proxy: Optional[str] = _UNSET,
- proxy_auth: Optional[Tuple[str, str]] = _UNSET,
- auth: Optional[Tuple[str, str]] = None,
- verify: Optional[bool] = _UNSET,
- cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
- http3: Optional[bool] = _UNSET,
- stealthy_headers: Optional[bool] = _UNSET,
- **kwargs,
- ) -> Response:
- return cast(
- Response,
- super().post(
- url,
- data,
- json,
- headers,
- params,
- cookies,
- timeout,
- follow_redirects,
- max_redirects,
- retries,
- retry_delay,
- proxies,
- proxy,
- proxy_auth,
- auth,
- verify,
- cert,
- impersonate,
- http3,
- stealthy_headers,
- **kwargs,
- ),
- )
+ async def __aenter__(self):
+ """Creates and returns a new asynchronous Session."""
+ if self._async_curl_session:
+ raise RuntimeError("This FetcherSession instance already has an active asynchronous session.")
- def put(
- self,
- url: str,
- data: Optional[Dict | str] = None,
- json: Optional[Dict | List] = None,
- headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
- params: Optional[Dict | List | Tuple] = None,
- cookies: Optional[CookieTypes] = None,
- timeout: Optional[int | float] = _UNSET,
- follow_redirects: Optional[bool] = _UNSET,
- max_redirects: Optional[int] = _UNSET,
- retries: Optional[int] = _UNSET,
- retry_delay: Optional[int] = _UNSET,
- proxies: Optional[ProxySpec] = _UNSET,
- proxy: Optional[str] = _UNSET,
- proxy_auth: Optional[Tuple[str, str]] = _UNSET,
- auth: Optional[Tuple[str, str]] = None,
- verify: Optional[bool] = _UNSET,
- cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
- http3: Optional[bool] = _UNSET,
- stealthy_headers: Optional[bool] = _UNSET,
- **kwargs,
- ) -> Response:
- return cast(
- Response,
- super().put(
- url,
- data,
- json,
- headers,
- params,
- cookies,
- timeout,
- follow_redirects,
- max_redirects,
- retries,
- retry_delay,
- proxies,
- proxy,
- proxy_auth,
- auth,
- verify,
- cert,
- impersonate,
- http3,
- stealthy_headers,
- **kwargs,
- ),
+ self._async_curl_session = AsyncCurlSession()
+ return self
+
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ """Closes the active asynchronous session managed by this instance, if any."""
+ # For type checking (not accessed error)
+ _ = (
+ exc_type,
+ exc_val,
+ exc_tb,
)
+ if self._async_curl_session:
+ await self._async_curl_session.close()
+ self._async_curl_session = None
- def delete(
+ async def __make_request(
self,
- url: str,
- data: Optional[Dict | str] = None,
- json: Optional[Dict | List] = None,
- headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
- params: Optional[Dict | List | Tuple] = None,
- cookies: Optional[CookieTypes] = None,
- timeout: Optional[int | float] = _UNSET,
- follow_redirects: Optional[bool] = _UNSET,
- max_redirects: Optional[int] = _UNSET,
- retries: Optional[int] = _UNSET,
- retry_delay: Optional[int] = _UNSET,
- proxies: Optional[ProxySpec] = _UNSET,
- proxy: Optional[str] = _UNSET,
- proxy_auth: Optional[Tuple[str, str]] = _UNSET,
- auth: Optional[Tuple[str, str]] = None,
- verify: Optional[bool] = _UNSET,
- cert: Optional[str | Tuple[str, str]] = _UNSET,
- impersonate: Optional[BrowserTypeLiteral] = _UNSET,
- http3: Optional[bool] = _UNSET,
- stealthy_headers: Optional[bool] = _UNSET,
+ method: SUPPORTED_HTTP_METHODS,
+ stealth: Optional[bool] = None,
**kwargs,
) -> Response:
- return cast(
- Response,
- super().delete(
- url,
- data,
- json,
- headers,
- params,
- cookies,
- timeout,
- follow_redirects,
- max_redirects,
- retries,
- retry_delay,
- proxies,
- proxy,
- proxy_auth,
- auth,
- verify,
- cert,
- impersonate,
- http3,
- stealthy_headers,
- **kwargs,
- ),
- )
+ """
+ Perform an HTTP request using the configured session.
+ :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
+ :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value.
+ :param kwargs: Additional request-specific arguments.
+ :return: A `Response` object.
+ """
+ stealth = self._stealth if stealth is None else stealth
-class AsyncFetcherClient(FetcherSession):
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self.__enter__: Any = None
- self.__exit__: Any = None
- self.__aenter__: Any = None
- self.__aexit__: Any = None
- self._async_curl_session: Any = True
+ selector_config = kwargs.pop("selector_config", {}) or self.selector_config
+ max_retries = self._get_with_precedence(kwargs.pop("retries"), self._default_retries)
+ retry_delay = self._get_with_precedence(kwargs.pop("retry_delay"), self._default_retry_delay)
+ request_args = self._merge_request_args(stealth=stealth, **kwargs)
+
+ session = self._async_curl_session
+ one_off_request = False
+ if session is _NO_SESSION and self.__aenter__ is None:
+ # For usage inside the ` AsyncFetcherClient ` class, and that's for several reasons
+ # 1. It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time.
+ # 2. `curl_cffi` doesn't support making async requests without sessions
+ # 3. Using a single session for many requests at the same time in async doesn't sit well with curl_cffi.
+ session = AsyncCurlSession()
+ one_off_request = True
+
+ if session:
+ for attempt in range(max_retries):
+ try:
+ response = await session.request(method, **request_args)
+ result = ResponseFactory.from_http_request(response, selector_config)
+ if one_off_request:
+ await session.close()
+ return result
+ except CurlError as e: # pragma: no cover
+ if attempt < max_retries - 1:
+ log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...")
+ await asyncio_sleep(retry_delay)
+ else:
+ log.error(f"Failed after {max_retries} attempts: {e}")
+ raise # Raise the exception if all retries fail
+ finally:
+ if session and one_off_request:
+ await session.close()
+
+ raise RuntimeError("No active session available.") # pragma: no cover
- # Setting the correct return types for the type checking/autocompletion
def get(
self,
url: str,
@@ -885,30 +705,54 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Awaitable[Response]:
- return cast(
- Awaitable[Response],
- super().get(
- url,
- params,
- headers,
- cookies,
- timeout,
- follow_redirects,
- max_redirects,
- retries,
- retry_delay,
- proxies,
- proxy,
- proxy_auth,
- auth,
- verify,
- cert,
- impersonate,
- http3,
- stealthy_headers,
- **kwargs,
- ),
+ """
+ Perform a GET request.
+
+ :param url: Target URL for the request.
+ :param params: Query string parameters for the request.
+ :param headers: Headers to include in the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
+ :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
+ :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
+ :return: A `Response` object.
+ """
+ method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")}
+ method_args.update(kwargs)
+ # For type checking (not accessed error)
+ _ = (
+ url,
+ params,
+ headers,
+ cookies,
+ timeout,
+ follow_redirects,
+ max_redirects,
+ retries,
+ retry_delay,
+ proxies,
+ proxy,
+ proxy_auth,
+ auth,
+ verify,
+ cert,
+ impersonate,
+ http3,
)
+ return self.__make_request("GET", stealth=stealthy_headers, **method_args)
def post(
self,
@@ -934,32 +778,58 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Awaitable[Response]:
- return cast(
- Awaitable[Response],
- super().post(
- url,
- data,
- json,
- headers,
- params,
- cookies,
- timeout,
- follow_redirects,
- max_redirects,
- retries,
- retry_delay,
- proxies,
- proxy,
- proxy_auth,
- auth,
- verify,
- cert,
- impersonate,
- http3,
- stealthy_headers,
- **kwargs,
- ),
+ """
+ Perform a POST request.
+
+ :param url: Target URL for the request.
+ :param data: Form data to include in the request body.
+ :param json: A JSON serializable object to include in the body of the request.
+ :param params: Query string parameters for the request.
+ :param headers: Headers to include in the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
+ :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
+ :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
+ :return: A `Response` object.
+ """
+ method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")}
+ method_args.update(kwargs)
+ # For type checking (not accessed error)
+ _ = (
+ url,
+ params,
+ headers,
+ data,
+ json,
+ cookies,
+ timeout,
+ follow_redirects,
+ max_redirects,
+ retries,
+ retry_delay,
+ proxies,
+ proxy,
+ proxy_auth,
+ auth,
+ verify,
+ cert,
+ impersonate,
+ http3,
)
+ return self.__make_request("POST", stealth=stealthy_headers, **method_args)
def put(
self,
@@ -985,32 +855,58 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Awaitable[Response]:
- return cast(
- Awaitable[Response],
- super().put(
- url,
- data,
- json,
- headers,
- params,
- cookies,
- timeout,
- follow_redirects,
- max_redirects,
- retries,
- retry_delay,
- proxies,
- proxy,
- proxy_auth,
- auth,
- verify,
- cert,
- impersonate,
- http3,
- stealthy_headers,
- **kwargs,
- ),
+ """
+ Perform a PUT request.
+
+ :param url: Target URL for the request.
+ :param data: Form data to include in the request body.
+ :param json: A JSON serializable object to include in the body of the request.
+ :param params: Query string parameters for the request.
+ :param headers: Headers to include in the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
+ :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
+ :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
+ :return: A `Response` object.
+ """
+ method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")}
+ method_args.update(kwargs)
+ # For type checking (not accessed error)
+ _ = (
+ url,
+ params,
+ headers,
+ data,
+ json,
+ cookies,
+ timeout,
+ follow_redirects,
+ max_redirects,
+ retries,
+ retry_delay,
+ proxies,
+ proxy,
+ proxy_auth,
+ auth,
+ verify,
+ cert,
+ impersonate,
+ http3,
)
+ return self.__make_request("PUT", stealth=stealthy_headers, **method_args)
def delete(
self,
@@ -1036,29 +932,174 @@ class AsyncFetcherClient(FetcherSession):
stealthy_headers: Optional[bool] = _UNSET,
**kwargs,
) -> Awaitable[Response]:
- return cast(
- Awaitable[Response],
- super().delete(
- url,
- data,
- json,
- headers,
- params,
- cookies,
- timeout,
- follow_redirects,
- max_redirects,
- retries,
- retry_delay,
- proxies,
- proxy,
- proxy_auth,
- auth,
- verify,
- cert,
- impersonate,
- http3,
- stealthy_headers,
- **kwargs,
- ),
+ """
+ Perform a DELETE request.
+
+ :param url: Target URL for the request.
+ :param data: Form data to include in the request body.
+ :param json: A JSON serializable object to include in the body of the request.
+ :param params: Query string parameters for the request.
+ :param headers: Headers to include in the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
+ :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
+ :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
+ :return: A `Response` object.
+ """
+ # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5,
+ # But some websites accept it, it depends on the implementation used.
+ method_args = {k: v for k, v in locals().items() if k not in ("self", "stealthy_headers", "kwargs")}
+ method_args.update(kwargs)
+ # For type checking (not accessed error)
+ _ = (
+ url,
+ params,
+ headers,
+ data,
+ json,
+ cookies,
+ timeout,
+ follow_redirects,
+ max_redirects,
+ retries,
+ retry_delay,
+ proxies,
+ proxy,
+ proxy_auth,
+ auth,
+ verify,
+ cert,
+ impersonate,
+ http3,
)
+ return self.__make_request("DELETE", stealth=stealthy_headers, **method_args)
+
+
+class FetcherSession:
+ """
+ A factory context manager that provides configured Fetcher sessions.
+
+ When this manager is used in a 'with' or 'async with' block,
+ it yields a new session configured with the manager's defaults.
+ A single instance of this manager should ideally be used for one active
+ session at a time (or sequentially). Re-entering a context with the
+ same manager instance while a session is already active is disallowed.
+ """
+
+ def __init__(
+ self,
+ impersonate: Optional[BrowserTypeLiteral] = "chrome",
+ http3: Optional[bool] = False,
+ stealthy_headers: Optional[bool] = True,
+ proxies: Optional[Dict[str, str]] = None,
+ proxy: Optional[str] = None,
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ timeout: Optional[int | float] = 30,
+ headers: Optional[Dict[str, str]] = None,
+ retries: Optional[int] = 3,
+ retry_delay: Optional[int] = 1,
+ follow_redirects: bool = True,
+ max_redirects: int = 30,
+ verify: bool = True,
+ cert: Optional[str | Tuple[str, str]] = None,
+ selector_config: Optional[Dict] = None,
+ ):
+ """
+ :param impersonate: Browser version to impersonate. Automatically defaults to the latest available Chrome version.
+ :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
+ :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
+ :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param timeout: Number of seconds to wait before timing out.
+ :param headers: Headers to include in the session with every request.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param verify: Whether to verify HTTPS certificates. Defaults to True.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param selector_config: Arguments passed when creating the final Selector class.
+ """
+ self._default_impersonate: Optional[BrowserTypeLiteral] = impersonate
+ self._stealth = stealthy_headers
+ self._default_proxies = proxies or {}
+ self._default_proxy = proxy or None
+ self._default_proxy_auth = proxy_auth or None
+ self._default_timeout = timeout
+ self._default_headers = headers or {}
+ self._default_retries = retries
+ self._default_retry_delay = retry_delay
+ self._default_follow_redirects = follow_redirects
+ self._default_max_redirects = max_redirects
+ self._default_verify = verify
+ self._default_cert = cert
+ self._default_http3 = http3
+ self.selector_config = selector_config or {}
+ self._client: _SyncSessionLogic | _ASyncSessionLogic | None = None
+
+ def __enter__(self) -> _SyncSessionLogic:
+ """Creates and returns a new synchronous Fetcher Session"""
+ if self._client is None:
+ # Use **vars(self) to avoid repeating all parameters
+ config = {k.replace("_default_", ""): v for k, v in vars(self).items() if k.startswith("_default")}
+ config["stealthy_headers"] = self._stealth
+ config["selector_config"] = self.selector_config
+ self._client = _SyncSessionLogic(**config)
+ return self._client.__enter__()
+ raise RuntimeError("This FetcherSession instance already has an active synchronous session.")
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ if self._client is not None and isinstance(self._client, _SyncSessionLogic):
+ self._client.__exit__(exc_type, exc_val, exc_tb)
+ self._client = None
+ return
+ raise RuntimeError("Cannot exit invalid session")
+
+ async def __aenter__(self) -> _ASyncSessionLogic:
+ """Creates and returns a new asynchronous Session."""
+ if self._client is None:
+ # Use **vars(self) to avoid repeating all parameters
+ config = {k.replace("_default_", ""): v for k, v in vars(self).items() if k.startswith("_default")}
+ config["stealthy_headers"] = self._stealth
+ config["selector_config"] = self.selector_config
+ self._client = _ASyncSessionLogic(**config)
+ return await self._client.__aenter__()
+ raise RuntimeError("This FetcherSession instance already has an active asynchronous session.")
+
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ if self._client is not None and isinstance(self._client, _ASyncSessionLogic):
+ await self._client.__aexit__(exc_type, exc_val, exc_tb)
+ self._client = None
+ return
+ raise RuntimeError("Cannot exit invalid session")
+
+
+class FetcherClient(_SyncSessionLogic):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.__enter__: Any = None
+ self.__exit__: Any = None
+ self._curl_session: Any = _NO_SESSION
+
+
+class AsyncFetcherClient(_ASyncSessionLogic):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.__aenter__: Any = None
+ self.__aexit__: Any = None
+ self._async_curl_session: Any = _NO_SESSION