Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(Fetcher): Replacing httpx + Adding FetcherSession

Karim shoair9e68e601613f
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py
index d4cfb37..da6574a 100644
--- a/scrapling/core/_types.py
+++ b/scrapling/core/_types.py
@@ -18,8 +18,11 @@ from typing import (
TypeVar,
Union,
Match,
+ Mapping,
+ Awaitable,
)
+SUPPORTED_HTTP_METHODS = Literal["GET", "POST", "PUT", "DELETE"]
SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
StrOrBytes = Union[str, bytes]
diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.py
index db9de24..5d0c240 100644
--- a/scrapling/engines/__init__.py
+++ b/scrapling/engines/__init__.py
@@ -1,7 +1,7 @@
from .camo import CamoufoxEngine
from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
from .pw import PlaywrightEngine
-from .static import StaticEngine
+from .static import FetcherSession, FetcherClient, AsyncFetcherClient
from .toolbelt import check_if_engine_usable
__all__ = ["CamoufoxEngine", "PlaywrightEngine"]
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 187d36e..58f87d8 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -1,61 +1,138 @@
-import httpx
-from httpx._models import Response as httpxResponse
+from time import sleep as time_sleep
+from asyncio import sleep as asyncio_sleep
-from scrapling.core._types import Dict, Optional, Tuple, Union
-from scrapling.core.utils import log, lru_cache
+from curl_cffi.requests.session import CurlError
+from curl_cffi.requests import (
+ ProxySpec,
+ CookieTypes,
+ BrowserTypeLiteral,
+ Session as CurlSession,
+ AsyncSession as AsyncCurlSession,
+)
-from .toolbelt import Response, generate_convincing_referer, generate_headers
+from scrapling.core.utils import log
+from scrapling.core._types import (
+ Dict,
+ Optional,
+ Tuple,
+ Union,
+ Mapping,
+ SUPPORTED_HTTP_METHODS,
+ Awaitable,
+ List,
+ Any,
+)
+from .toolbelt import (
+ Response,
+ generate_convincing_referer,
+ generate_headers,
+ ResponseFactory,
+)
+
+__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")
+
+
+class FetcherSession:
+ """
+ A context manager that provides configured Fetcher sessions.
+
+ When this manager is used in a 'with' or 'async with' block,
+ it yields a new session configured with the manager's defaults.
+ A single instance of this manager should ideally be used for one active
+ session at a time (or sequentially). Re-entering a context with the
+ same manager instance while a session is already active is disallowed.
+ """
-@lru_cache(2, typed=True) # Singleton easily
-class StaticEngine:
def __init__(
self,
- url: str,
+ impersonate: Optional[str] = "chrome136",
+ stealthy_headers: Optional[bool] = True,
+ proxies: Optional[Dict[str, str]] = None,
proxy: Optional[str] = None,
- stealthy_headers: bool = True,
- follow_redirects: bool = True,
- timeout: Optional[Union[int, float]] = None,
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ timeout: Optional[Union[int, float]] = 30,
+ headers: Optional[Dict[str, str]] = None,
retries: Optional[int] = 3,
- cookies: Optional[Tuple] = None,
- adaptor_arguments: Tuple = None,
+ retry_delay: Optional[int] = 1,
+ follow_redirects: bool = True,
+ max_redirects: int = 30,
+ verify: bool = True,
+ cert: Optional[Union[str, Tuple[str, str]]] = None,
+ adaptor_arguments: Optional[Dict] = None,
):
- """An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
-
- :param url: Target url.
- :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
- create a referer header as if this request had came from Google's search of this URL's domain.
- :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
- :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
- :param cookies: Set cookies for the next request.
- :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
- :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
- """
- self.url = url
- self.proxy = proxy
+ """
+ :param impersonate: Browser version to impersonate. Defaults to "chrome136".
+ :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
+ :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param timeout: Number of seconds to wait before timing out.
+ :param headers: Headers to include in the session with every request.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param verify: Whether to verify HTTPS certificates. Defaults to True.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param adaptor_arguments: Arguments passed when creating the final Adaptor class.
+ """
+ self.default_impersonate = impersonate
self.stealth = stealthy_headers
- self.timeout = timeout
- self.follow_redirects = bool(follow_redirects)
- self.retries = retries
- self.cookies = dict(cookies) if cookies else {}
- self._extra_headers = generate_headers(browser_mode=False)
- # Because we are using `lru_cache` for a slight optimization but both dict/dict_items are not hashable so they can't be cached
- # So my solution here was to convert it to tuple then convert it back to dictionary again here as tuples are hashable, ofc `tuple().__hash__()`
- self.adaptor_arguments = dict(adaptor_arguments) if adaptor_arguments else {}
-
- def _headers_job(self, headers: Optional[Dict]) -> Dict:
+ self.default_proxies = proxies or {}
+ self.default_proxy = proxy or None
+ self.default_proxy_auth = proxy_auth or None
+ self.default_timeout = timeout
+ self.default_headers = headers or {}
+ self.default_retries = retries
+ self.default_retry_delay = retry_delay
+ self.default_follow_redirects = follow_redirects
+ self.default_max_redirects = max_redirects
+ self.default_verify = verify
+ self.default_cert = cert
+ self.adaptor_arguments = adaptor_arguments or {}
+
+ self._curl_session: Optional[CurlSession] = None
+ self._async_curl_session: Optional[AsyncCurlSession] = None
+
+ def _merge_request_args(self, **kwargs) -> Dict[str, Any]:
+ """Merge request-specific arguments with default session arguments."""
+ request_args = {
+ "headers": self._headers_job(
+ kwargs["url"], kwargs.get("headers"), kwargs.pop("stealth")
+ ),
+ "proxies": kwargs.get("proxies", self.default_proxies),
+ "proxy": kwargs.get("proxy", self.default_proxy),
+ "proxy_auth": kwargs.get("proxy_auth", self.default_proxy_auth),
+ "timeout": kwargs.get("timeout", self.default_timeout),
+ "allow_redirects": kwargs.get(
+ "follow_redirects", self.default_follow_redirects
+ ),
+ "max_redirects": kwargs.get("max_redirects", self.default_max_redirects),
+ "verify": kwargs.get("verify", self.default_verify),
+ "cert": kwargs.get("cert", self.default_cert),
+ "impersonate": kwargs.get("impersonate", self.default_impersonate),
+ **kwargs,
+ }
+ return request_args
+
+ def _headers_job(
+ self, url, headers: Optional[Dict], stealth: Optional[bool]
+ ) -> Dict:
"""Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and
finally generates a referer header that looks like if this request came from Google's search of the current URL's domain.
:param headers: Current headers in the request if the user passed any
+ :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value.
:return: A dictionary of the new headers.
"""
- headers = headers or {}
+ headers = {**self.default_headers, **(headers or {})}
headers_keys = set(map(str.lower, headers.keys()))
- if self.stealth:
+ if stealth:
extra_headers = generate_headers(browser_mode=False)
- # Don't overwrite user supplied headers
+ # Don't overwrite user-supplied headers
extra_headers = {
key: value
for key, value in extra_headers.items()
@@ -63,133 +140,772 @@ class StaticEngine:
}
headers.update(extra_headers)
if "referer" not in headers_keys:
- headers.update({"referer": generate_convincing_referer(self.url)})
+ headers.update({"referer": generate_convincing_referer(url)})
elif "user-agent" not in headers_keys:
- headers["User-Agent"] = generate_headers(browser_mode=False).get(
- "User-Agent"
- )
+ headers["User-Agent"] = __default_useragent__
log.debug(
f"Can't find useragent in headers so '{headers['User-Agent']}' was used."
)
return headers
- def _prepare_response(self, response: httpxResponse) -> Response:
- """Takes httpx response and generates `Response` object from it.
-
- :param response: httpx response object
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
- """
- return Response(
- url=str(response.url),
- text=response.text,
- body=response.content,
- status=response.status_code,
- reason=response.reason_phrase,
- encoding=response.encoding or "utf-8",
- cookies=dict(response.cookies),
- headers=dict(response.headers),
- request_headers=dict(response.request.headers),
- method=response.request.method,
- history=[
- self._prepare_response(redirection) for redirection in response.history
- ],
- **self.adaptor_arguments,
- )
+ def __enter__(self):
+ """Creates and returns a new synchronous Fetcher Session"""
+ if self._curl_session:
+ raise RuntimeError(
+ "This FetcherSession instance already has an active synchronous session. "
+ "Create a new FetcherSession instance for a new independent session, "
+ "or use the current instance sequentially after the previous context has exited."
+ )
+ if (
+ self._async_curl_session
+ ): # Prevent mixing if async is active from this instance
+ raise RuntimeError(
+ "This FetcherSession instance has an active asynchronous session. "
+ "Cannot enter a synchronous context simultaneously with the same manager instance."
+ )
+
+ self._curl_session = CurlSession()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ """Closes the active synchronous session managed by this instance, if any."""
+ if self._curl_session:
+ self._curl_session.close()
+ self._curl_session = None
+
+ async def __aenter__(self):
+ """Creates and returns a new asynchronous Session."""
+ if self._async_curl_session:
+ raise RuntimeError(
+ "This FetcherSession instance already has an active asynchronous session. "
+ "Create a new FetcherSession instance for a new independent session, "
+ "or use the current instance sequentially after the previous context has exited."
+ )
+ if self._curl_session: # Prevent mixing if sync is active from this instance
+ raise RuntimeError(
+ "This FetcherSession instance has an active synchronous session. "
+ "Cannot enter an asynchronous context simultaneously with the same manager instance."
+ )
+
+ self._async_curl_session = AsyncCurlSession()
+ return self
+
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ """Closes the active asynchronous session managed by this instance, if any."""
+ if self._async_curl_session:
+ await self._async_curl_session.close()
+ self._async_curl_session = None
+
+ def __make_request(
+ self,
+ method: SUPPORTED_HTTP_METHODS,
+ request_args: Dict[str, Any],
+ max_retries: int,
+ retry_delay: int,
+ adaptor_arguments: Optional[Dict] = None,
+ ) -> Response:
+ """
+ Perform an HTTP request using the configured session.
+
+ :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
+ :param url: Target URL for the request.
+ :param request_args: Arguments to be passed to the session's `request()` method.
+ :param max_retries: Maximum number of retries for the request.
+ :param retry_delay: Number of seconds to wait between retries.
+ :param adaptor_arguments: Arguments passed when creating the final Adaptor class.
+ :return: A `Response` object for synchronous requests or an awaitable for asynchronous.
+ """
+ if self._curl_session:
+ for attempt in range(max_retries):
+ try:
+ response = self._curl_session.request(method, **request_args)
+ # response.raise_for_status() # Retry responses with a status code between 200-400
+ return ResponseFactory.from_http_request(
+ response, adaptor_arguments
+ )
+ except CurlError as e:
+ if attempt < max_retries - 1:
+ log.error(
+ f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..."
+ )
+ time_sleep(retry_delay)
+ else:
+ log.error(f"Failed after {max_retries} attempts: {e}")
+ raise # Raise the exception if all retries fail
+
+ raise RuntimeError("No active session available.")
+
+ async def __make_async_request(
+ self,
+ method: SUPPORTED_HTTP_METHODS,
+ request_args: Dict[str, Any],
+ max_retries: int,
+ retry_delay: int,
+ adaptor_arguments: Optional[Dict] = None,
+ ) -> Response:
+ """
+ Perform an HTTP request using the configured session.
+
+ :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
+ :param url: Target URL for the request.
+ :param request_args: Arguments to be passed to the session's `request()` method.
+ :param max_retries: Maximum number of retries for the request.
+ :param retry_delay: Number of seconds to wait between retries.
+ :param adaptor_arguments: Arguments passed when creating the final Adaptor class.
+ :return: A `Response` object for synchronous requests or an awaitable for asynchronous.
+ """
+ if self._async_curl_session:
+ for attempt in range(max_retries):
+ try:
+ response = await self._async_curl_session.request(
+ method, **request_args
+ )
+ # response.raise_for_status() # Retry responses with a status code between 200-400
+ return ResponseFactory.from_http_request(
+ response, adaptor_arguments
+ )
+ except CurlError as e:
+ if attempt < max_retries - 1:
+ log.error(
+ f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..."
+ )
+ await asyncio_sleep(retry_delay)
+ else:
+ log.error(f"Failed after {max_retries} attempts: {e}")
+ raise # Raise the exception if all retries fail
+
+ raise RuntimeError("No active session available.")
+
+ def __prepare_and_dispatch(
+ self,
+ method: SUPPORTED_HTTP_METHODS,
+ stealth: Optional[bool] = None,
+ **kwargs,
+ ) -> Union[Response, Awaitable[Response]]:
+ """
+ Internal dispatcher. Prepares arguments and calls sync or async request helper.
+
+ :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
+ :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value.
+ :param url: Target URL for the request.
+ :param kwargs: Additional request-specific arguments.
+ :return: A `Response` object for synchronous requests or an awaitable for asynchronous.
+ """
+ stealth = self.stealth if stealth is None else stealth
- def _make_request(self, method: str, **kwargs) -> Response:
- headers = self._headers_job(kwargs.pop("headers", {}))
- with httpx.Client(
- proxy=self.proxy,
- transport=httpx.HTTPTransport(retries=self.retries),
- cookies=self.cookies,
- ) as client:
- request = getattr(client, method)(
- url=self.url,
- headers=headers,
- follow_redirects=self.follow_redirects,
- timeout=self.timeout,
- **kwargs,
+ adaptor_arguments = (
+ kwargs.pop("adaptor_arguments", {}) or self.adaptor_arguments
+ )
+ max_retries = kwargs.pop("retries", self.default_retries)
+ retry_delay = kwargs.pop("retry_delay", self.default_retry_delay)
+ request_args = self._merge_request_args(stealth=stealth, **kwargs)
+ if self._curl_session:
+ return self.__make_request(
+ method, request_args, max_retries, retry_delay, adaptor_arguments
)
- return self._prepare_response(request)
-
- async def _async_make_request(self, method: str, **kwargs) -> Response:
- headers = self._headers_job(kwargs.pop("headers", {}))
- async with httpx.AsyncClient(
- proxy=self.proxy,
- transport=httpx.AsyncHTTPTransport(retries=self.retries),
- cookies=self.cookies,
- ) as client:
- request = await getattr(client, method)(
- url=self.url,
- headers=headers,
- follow_redirects=self.follow_redirects,
- timeout=self.timeout,
- **kwargs,
+ elif self._async_curl_session:
+ # The returned value is a Coroutine
+ return self.__make_async_request(
+ method, request_args, max_retries, retry_delay, adaptor_arguments
)
- return self._prepare_response(request)
- def get(self, **kwargs: Dict) -> Response:
- """Make basic HTTP GET request for you but with some added flavors.
+ raise RuntimeError("No active session available.")
- :param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+ def get(
+ self,
+ url: str,
+ params: Optional[Union[Dict, List, Tuple]] = None, # <--
+ headers: Optional[Mapping[str, Optional[str]]] = None,
+ cookies: Optional[CookieTypes] = None, # <--
+ timeout: Optional[Union[int, float]] = 30, # <--
+ follow_redirects: Optional[bool] = True, # <--
+ max_redirects: Optional[int] = 30, # <--
+ retries: Optional[int] = 3,
+ retry_delay: Optional[int] = 1, # <--
+ proxies: Optional[ProxySpec] = None, # <--
+ proxy: Optional[str] = None, # <--
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ auth: Optional[Tuple[str, str]] = None,
+ verify: Optional[bool] = True, # <--
+ cert: Optional[Union[str, Tuple[str, str]]] = None,
+ impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
+ stealthy_headers: Optional[bool] = True,
+ **kwargs,
+ ) -> Union[Response, Awaitable[Response]]:
"""
- return self._make_request("get", **kwargs)
+ Perform a GET request.
- async def async_get(self, **kwargs: Dict) -> Response:
- """Make basic async HTTP GET request for you but with some added flavors.
+ :param url: Target URL for the request.
+ :param params: Query string parameters for the request.
+ :param headers: Headers to include in the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Defaults to "chrome136".
+ :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
+ :return: A `Response` object or an awaitable for async.
+ """
+ request_args = {
+ "url": url,
+ "params": params,
+ "headers": headers,
+ "cookies": cookies,
+ "timeout": timeout,
+ "retry_delay": retry_delay,
+ "allow_redirects": follow_redirects,
+ "max_redirects": max_redirects,
+ "retries": retries,
+ "proxies": proxies,
+ "proxy": proxy,
+ "proxy_auth": proxy_auth,
+ "auth": auth,
+ "verify": verify,
+ "cert": cert,
+ "impersonate": impersonate,
+ **kwargs,
+ }
+ return self.__prepare_and_dispatch(
+ "GET", stealth=stealthy_headers, **request_args
+ )
- :param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+ def post(
+ self,
+ url: str,
+ data: Optional[Union[Dict, str]] = None,
+ json: Optional[Union[Dict, List]] = None,
+ headers: Optional[Mapping[str, Optional[str]]] = None,
+ params: Optional[Union[Dict, List, Tuple]] = None, # <--
+ cookies: Optional[CookieTypes] = None, # <--
+ timeout: Optional[Union[int, float]] = 30, # <--
+ follow_redirects: Optional[bool] = True, # <--
+ max_redirects: Optional[int] = 30, # <--
+ retries: Optional[int] = 3,
+ retry_delay: Optional[int] = 1, # <--
+ proxies: Optional[ProxySpec] = None, # <--
+ proxy: Optional[str] = None, # <--
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ auth: Optional[Tuple[str, str]] = None,
+ verify: Optional[bool] = True, # <--
+ cert: Optional[Union[str, Tuple[str, str]]] = None,
+ impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
+ stealthy_headers: Optional[bool] = True,
+ **kwargs,
+ ) -> Union[Response, Awaitable[Response]]:
"""
- return await self._async_make_request("get", **kwargs)
+ Perform a POST request.
- def post(self, **kwargs: Dict) -> Response:
- """Make basic HTTP POST request for you but with some added flavors.
+ :param url: Target URL for the request.
+ :param data: Form data to include in the request body.
+ :param json: A JSON serializable object to include in the body of the request.
+ :param headers: Headers to include in the request.
+ :param params: Query string parameters for the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates. Defaults to True.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Defaults to "chrome136".
+ :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
+ :return: A `Response` object or an awaitable for async.
+ """
+ request_args = {
+ "url": url,
+ "data": data,
+ "json": json,
+ "headers": headers,
+ "params": params,
+ "cookies": cookies,
+ "timeout": timeout,
+ "retry_delay": retry_delay,
+ "proxy": proxy,
+ "impersonate": impersonate,
+ "allow_redirects": follow_redirects,
+ "max_redirects": max_redirects,
+ "retries": retries,
+ "proxies": proxies,
+ "proxy_auth": proxy_auth,
+ "auth": auth,
+ "verify": verify,
+ "cert": cert,
+ **kwargs,
+ }
+ return self.__prepare_and_dispatch(
+ "POST", stealth=stealthy_headers, **request_args
+ )
- :param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+ def put(
+ self,
+ url: str,
+ data: Optional[Union[Dict, str]] = None,
+ json: Optional[Union[Dict, List]] = None,
+ headers: Optional[Mapping[str, Optional[str]]] = None,
+ params: Optional[Union[Dict, List, Tuple]] = None, # <--
+ cookies: Optional[CookieTypes] = None, # <--
+ timeout: Optional[Union[int, float]] = 30, # <--
+ follow_redirects: Optional[bool] = True, # <--
+ max_redirects: Optional[int] = 30, # <--
+ retries: Optional[int] = 3,
+ retry_delay: Optional[int] = 1, # <--
+ proxies: Optional[ProxySpec] = None, # <--
+ proxy: Optional[str] = None, # <--
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ auth: Optional[Tuple[str, str]] = None,
+ verify: Optional[bool] = True, # <--
+ cert: Optional[Union[str, Tuple[str, str]]] = None,
+ impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
+ stealthy_headers: Optional[bool] = True,
+ **kwargs,
+ ) -> Union[Response, Awaitable[Response]]:
"""
- return self._make_request("post", **kwargs)
+ Perform a PUT request.
- async def async_post(self, **kwargs: Dict) -> Response:
- """Make basic async HTTP POST request for you but with some added flavors.
+ :param url: Target URL for the request.
+ :param data: Form data to include in the request body.
+ :param json: A JSON serializable object to include in the body of the request.
+ :param headers: Headers to include in the request.
+ :param params: Query string parameters for the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates. Defaults to True.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Defaults to "chrome136".
+ :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
+ :return: A `Response` object or an awaitable for async.
+ """
+ request_args = {
+ "url": url,
+ "data": data,
+ "json": json,
+ "headers": headers,
+ "params": params,
+ "cookies": cookies,
+ "timeout": timeout,
+ "retry_delay": retry_delay,
+ "proxy": proxy,
+ "impersonate": impersonate,
+ "allow_redirects": follow_redirects,
+ "max_redirects": max_redirects,
+ "retries": retries,
+ "proxies": proxies,
+ "proxy_auth": proxy_auth,
+ "auth": auth,
+ "verify": verify,
+ "cert": cert,
+ **kwargs,
+ }
+ return self.__prepare_and_dispatch(
+ "PUT", stealth=stealthy_headers, **request_args
+ )
- :param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+ def delete(
+ self,
+ url: str,
+ data: Optional[Union[Dict, str]] = None,
+ json: Optional[Union[Dict, List]] = None,
+ headers: Optional[Mapping[str, Optional[str]]] = None,
+ params: Optional[Union[Dict, List, Tuple]] = None, # <--
+ cookies: Optional[CookieTypes] = None, # <--
+ timeout: Optional[Union[int, float]] = 30, # <--
+ follow_redirects: Optional[bool] = True, # <--
+ max_redirects: Optional[int] = 30, # <--
+ retries: Optional[int] = 3,
+ retry_delay: Optional[int] = 1, # <--
+ proxies: Optional[ProxySpec] = None, # <--
+ proxy: Optional[str] = None, # <--
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ auth: Optional[Tuple[str, str]] = None,
+ verify: Optional[bool] = True, # <--
+ cert: Optional[Union[str, Tuple[str, str]]] = None,
+ impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
+ stealthy_headers: Optional[bool] = True,
+ **kwargs,
+ ) -> Union[Response, Awaitable[Response]]:
"""
- return await self._async_make_request("post", **kwargs)
+ Perform a DELETE request.
+
+ :param url: Target URL for the request.
+ :param data: Form data to include in the request body.
+ :param json: A JSON serializable object to include in the body of the request.
+ :param headers: Headers to include in the request.
+ :param params: Query string parameters for the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates. Defaults to True.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Defaults to "chrome136".
+ :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method.
+ :return: A `Response` object or an awaitable for async.
+ """
+ request_args = {
+ "url": url,
+ # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5,
+ # But some websites accept it, it depends on the implementation used.
+ "data": data,
+ "json": json,
+ "headers": headers,
+ "params": params,
+ "cookies": cookies,
+ "timeout": timeout,
+ "retry_delay": retry_delay,
+ "proxy": proxy,
+ "impersonate": impersonate,
+ "allow_redirects": follow_redirects,
+ "max_redirects": max_redirects,
+ "retries": retries,
+ "proxies": proxies,
+ "proxy_auth": proxy_auth,
+ "auth": auth,
+ "verify": verify,
+ "cert": cert,
+ **kwargs,
+ }
+ return self.__prepare_and_dispatch(
+ "DELETE", stealth=stealthy_headers, **request_args
+ )
+
+
+class FetcherClient(FetcherSession):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ # Using one session for all requests is faster than using stateless `curl_cffi.get`
+ self.__enter__ = None
+ self.__exit__ = None
+ self.__aenter__ = None
+ self.__aexit__ = None
+ self._curl_session = CurlSession()
- def delete(self, **kwargs: Dict) -> Response:
- """Make basic HTTP DELETE request for you but with some added flavors.
- :param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+class AsyncFetcherClient:
+ # Since curl_cffi doesn't support making async requests without sessions
+ # And using a single session for many requests at the same time in async doesn't sit well with curl_cffi.
+ # We do this
+
+ @staticmethod
+ async def get(
+ url: str,
+ params: Optional[Union[Dict, List, Tuple]] = None, # <--
+ headers: Optional[Mapping[str, Optional[str]]] = None,
+ cookies: Optional[CookieTypes] = None, # <--
+ timeout: Optional[Union[int, float]] = 30, # <--
+ follow_redirects: Optional[bool] = True, # <--
+ max_redirects: Optional[int] = 30, # <--
+ retries: Optional[int] = 3,
+ retry_delay: Optional[int] = 1, # <--
+ proxies: Optional[ProxySpec] = None, # <--
+ proxy: Optional[str] = None, # <--
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ auth: Optional[Tuple[str, str]] = None,
+ verify: Optional[bool] = True, # <--
+ cert: Optional[Union[str, Tuple[str, str]]] = None,
+ impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
+ stealthy_headers: Optional[bool] = True,
+ **kwargs,
+ ) -> Response:
"""
- return self._make_request("delete", **kwargs)
+ Perform a GET request.
- async def async_delete(self, **kwargs: Dict) -> Response:
- """Make basic async HTTP DELETE request for you but with some added flavors.
+ :param url: Target URL for the request.
+ :param params: Query string parameters for the request.
+ :param headers: Headers to include in the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Defaults to "chrome136".
+ :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
+ :return: An awaitable `Response` object.
+ """
+ request_args = {
+ "url": url,
+ "params": params,
+ "headers": headers,
+ "cookies": cookies,
+ "timeout": timeout,
+ "retry_delay": retry_delay,
+ "allow_redirects": follow_redirects,
+ "max_redirects": max_redirects,
+ "retries": retries,
+ "proxies": proxies,
+ "proxy": proxy,
+ "proxy_auth": proxy_auth,
+ "auth": auth,
+ "verify": verify,
+ "cert": cert,
+ "impersonate": impersonate,
+ **kwargs,
+ }
+ async with FetcherSession(stealthy_headers=stealthy_headers) as client:
+ return await client.get(**request_args)
- :param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+ @staticmethod
+ async def post(
+ url: str,
+ data: Optional[Union[Dict, str]] = None,
+ json: Optional[Union[Dict, List]] = None,
+ headers: Optional[Mapping[str, Optional[str]]] = None,
+ params: Optional[Union[Dict, List, Tuple]] = None, # <--
+ cookies: Optional[CookieTypes] = None, # <--
+ timeout: Optional[Union[int, float]] = 30, # <--
+ follow_redirects: Optional[bool] = True, # <--
+ max_redirects: Optional[int] = 30, # <--
+ retries: Optional[int] = 3,
+ retry_delay: Optional[int] = 1, # <--
+ proxies: Optional[ProxySpec] = None, # <--
+ proxy: Optional[str] = None, # <--
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ auth: Optional[Tuple[str, str]] = None,
+ verify: Optional[bool] = True, # <--
+ cert: Optional[Union[str, Tuple[str, str]]] = None,
+ impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
+ stealthy_headers: Optional[bool] = True,
+ **kwargs,
+ ) -> Response:
"""
- return await self._async_make_request("delete", **kwargs)
+ Perform a POST request.
- def put(self, **kwargs: Dict) -> Response:
- """Make basic HTTP PUT request for you but with some added flavors.
+ :param url: Target URL for the request.
+ :param data: Form data to include in the request body.
+ :param json: A JSON serializable object to include in the body of the request.
+ :param headers: Headers to include in the request.
+ :param params: Query string parameters for the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates. Defaults to True.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Defaults to "chrome136".
+ :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
+ :return: An awaitable `Response` object.
+ """
+ request_args = {
+ "url": url,
+ "data": data,
+ "json": json,
+ "headers": headers,
+ "params": params,
+ "cookies": cookies,
+ "timeout": timeout,
+ "retry_delay": retry_delay,
+ "proxy": proxy,
+ "impersonate": impersonate,
+ "allow_redirects": follow_redirects,
+ "max_redirects": max_redirects,
+ "retries": retries,
+ "proxies": proxies,
+ "proxy_auth": proxy_auth,
+ "auth": auth,
+ "verify": verify,
+ "cert": cert,
+ **kwargs,
+ }
+ async with FetcherSession(stealthy_headers=stealthy_headers) as client:
+ return await client.post(**request_args)
- :param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+ @staticmethod
+ async def put(
+ url: str,
+ data: Optional[Union[Dict, str]] = None,
+ json: Optional[Union[Dict, List]] = None,
+ headers: Optional[Mapping[str, Optional[str]]] = None,
+ params: Optional[Union[Dict, List, Tuple]] = None, # <--
+ cookies: Optional[CookieTypes] = None, # <--
+ timeout: Optional[Union[int, float]] = 30, # <--
+ follow_redirects: Optional[bool] = True, # <--
+ max_redirects: Optional[int] = 30, # <--
+ retries: Optional[int] = 3,
+ retry_delay: Optional[int] = 1, # <--
+ proxies: Optional[ProxySpec] = None, # <--
+ proxy: Optional[str] = None, # <--
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ auth: Optional[Tuple[str, str]] = None,
+ verify: Optional[bool] = True, # <--
+ cert: Optional[Union[str, Tuple[str, str]]] = None,
+ impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
+ stealthy_headers: Optional[bool] = True,
+ **kwargs,
+ ) -> Response:
"""
- return self._make_request("put", **kwargs)
+ Perform a PUT request.
- async def async_put(self, **kwargs: Dict) -> Response:
- """Make basic async HTTP PUT request for you but with some added flavors.
+ :param url: Target URL for the request.
+ :param data: Form data to include in the request body.
+ :param json: A JSON serializable object to include in the body of the request.
+ :param headers: Headers to include in the request.
+ :param params: Query string parameters for the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates. Defaults to True.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Defaults to "chrome136".
+ :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
+ :return: An awaitable `Response` object.
+ """
+ request_args = {
+ "url": url,
+ "data": data,
+ "json": json,
+ "headers": headers,
+ "params": params,
+ "cookies": cookies,
+ "timeout": timeout,
+ "retry_delay": retry_delay,
+ "proxy": proxy,
+ "impersonate": impersonate,
+ "allow_redirects": follow_redirects,
+ "max_redirects": max_redirects,
+ "retries": retries,
+ "proxies": proxies,
+ "proxy_auth": proxy_auth,
+ "auth": auth,
+ "verify": verify,
+ "cert": cert,
+ **kwargs,
+ }
+ async with FetcherSession(stealthy_headers=stealthy_headers) as client:
+ return await client.put(**request_args)
+
+ @staticmethod
+ async def delete(
+ url: str,
+ data: Optional[Union[Dict, str]] = None,
+ json: Optional[Union[Dict, List]] = None,
+ headers: Optional[Mapping[str, Optional[str]]] = None,
+ params: Optional[Union[Dict, List, Tuple]] = None, # <--
+ cookies: Optional[CookieTypes] = None, # <--
+ timeout: Optional[Union[int, float]] = 30, # <--
+ follow_redirects: Optional[bool] = True, # <--
+ max_redirects: Optional[int] = 30, # <--
+ retries: Optional[int] = 3,
+ retry_delay: Optional[int] = 1, # <--
+ proxies: Optional[ProxySpec] = None, # <--
+ proxy: Optional[str] = None, # <--
+ proxy_auth: Optional[Tuple[str, str]] = None,
+ auth: Optional[Tuple[str, str]] = None,
+ verify: Optional[bool] = True, # <--
+ cert: Optional[Union[str, Tuple[str, str]]] = None,
+ impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <--
+ stealthy_headers: Optional[bool] = True,
+ **kwargs,
+ ) -> Response:
+ """
+ Perform a DELETE request.
- :param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
+ :param url: Target URL for the request.
+ :param data: Form data to include in the request body.
+ :param json: A JSON serializable object to include in the body of the request.
+ :param headers: Headers to include in the request.
+ :param params: Query string parameters for the request.
+ :param cookies: Cookies to use in the request.
+ :param timeout: Number of seconds to wait before timing out.
+ :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
+ :param retries: Number of retry attempts. Defaults to 3.
+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
+ :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}.
+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
+ Cannot be used together with the `proxies` parameter.
+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param verify: Whether to verify HTTPS certificates. Defaults to True.
+ :param cert: Tuple of (cert, key) filenames for the client certificate.
+ :param impersonate: Browser version to impersonate. Defaults to "chrome136".
+ :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain.
+ :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method.
+ :return: An awaitable `Response` object.
"""
- return await self._async_make_request("put", **kwargs)
+ request_args = {
+ "url": url,
+ # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5,
+ # But some websites accept it, it depends on the implementation used.
+ "data": data,
+ "json": json,
+ "headers": headers,
+ "params": params,
+ "cookies": cookies,
+ "timeout": timeout,
+ "retry_delay": retry_delay,
+ "proxy": proxy,
+ "impersonate": impersonate,
+ "allow_redirects": follow_redirects,
+ "max_redirects": max_redirects,
+ "retries": retries,
+ "proxies": proxies,
+ "proxy_auth": proxy_auth,
+ "auth": auth,
+ "verify": verify,
+ "cert": cert,
+ **kwargs,
+ }
+ async with FetcherSession(stealthy_headers=stealthy_headers) as client:
+ return await client.delete(**request_args)
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index bbd4b9f..b20b0a1 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -9,462 +9,34 @@ from scrapling.core._types import (
Iterable,
)
from scrapling.engines import (
+ FetcherSession,
CamoufoxEngine,
PlaywrightEngine,
- StaticEngine,
check_if_engine_usable,
+ FetcherClient as _FetcherClient,
+ AsyncFetcherClient as _AsyncFetcherClient,
)
from scrapling.engines.toolbelt import BaseFetcher, Response
+__FetcherClientInstance__ = _FetcherClient()
-class Fetcher(BaseFetcher):
- """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on httpx.
-
- Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly.
- """
-
- @classmethod
- def get(
- cls,
- url: str,
- follow_redirects: bool = True,
- timeout: Optional[Union[int, float]] = 10,
- stealthy_headers: bool = True,
- proxy: Optional[str] = None,
- retries: Optional[int] = 3,
- cookies: Optional[Dict] = None,
- custom_config: Dict = None,
- **kwargs: Dict,
- ) -> Response:
- """Make basic HTTP GET request for you but with some added flavors.
-
- :param url: Target url.
- :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
- :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
- :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
- create a referer header as if this request had came from Google's search of this URL's domain.
- :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
- :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
- :param cookies: Set cookies for the next request.
- :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
- :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
- """
- if not custom_config:
- custom_config = {}
- elif not isinstance(custom_config, dict):
- ValueError(
- f"The custom parser config must be of type dictionary, got {cls.__class__}"
- )
-
- adaptor_arguments = tuple(
- {**cls._generate_parser_arguments(), **custom_config}.items()
- )
-
- if not cookies:
- cookies = {}
- elif not isinstance(cookies, dict):
- ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
- response_object = StaticEngine(
- url,
- proxy,
- stealthy_headers,
- follow_redirects,
- timeout,
- retries,
- tuple(cookies.items()),
- adaptor_arguments=adaptor_arguments,
- ).get(**kwargs)
- return response_object
-
- @classmethod
- def post(
- cls,
- url: str,
- follow_redirects: bool = True,
- timeout: Optional[Union[int, float]] = 10,
- stealthy_headers: bool = True,
- proxy: Optional[str] = None,
- retries: Optional[int] = 3,
- cookies: Optional[Dict] = None,
- custom_config: Dict = None,
- **kwargs: Dict,
- ) -> Response:
- """Make basic HTTP POST request for you but with some added flavors.
-
- :param url: Target url.
- :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
- :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
- :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
- create a referer header as if this request came from Google's search of this URL's domain.
- :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
- :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
- :param cookies: Set cookies for the next request.
- :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
- :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
- """
- if not custom_config:
- custom_config = {}
- elif not isinstance(custom_config, dict):
- ValueError(
- f"The custom parser config must be of type dictionary, got {cls.__class__}"
- )
- adaptor_arguments = tuple(
- {**cls._generate_parser_arguments(), **custom_config}.items()
- )
-
- if not cookies:
- cookies = {}
- elif not isinstance(cookies, dict):
- ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
- response_object = StaticEngine(
- url,
- proxy,
- stealthy_headers,
- follow_redirects,
- timeout,
- retries,
- tuple(cookies.items()),
- adaptor_arguments=adaptor_arguments,
- ).post(**kwargs)
- return response_object
-
- @classmethod
- def put(
- cls,
- url: str,
- follow_redirects: bool = True,
- timeout: Optional[Union[int, float]] = 10,
- stealthy_headers: bool = True,
- proxy: Optional[str] = None,
- retries: Optional[int] = 3,
- cookies: Optional[Dict] = None,
- custom_config: Dict = None,
- **kwargs: Dict,
- ) -> Response:
- """Make basic HTTP PUT request for you but with some added flavors.
-
- :param url: Target url
- :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
- :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
- :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
- create a referer header as if this request came from Google's search of this URL's domain.
- :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
- :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
- :param cookies: Set cookies for the next request.
- :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
- :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
-
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
- """
- if not custom_config:
- custom_config = {}
- elif not isinstance(custom_config, dict):
- ValueError(
- f"The custom parser config must be of type dictionary, got {cls.__class__}"
- )
-
- adaptor_arguments = tuple(
- {**cls._generate_parser_arguments(), **custom_config}.items()
- )
-
- if not cookies:
- cookies = {}
- elif not isinstance(cookies, dict):
- ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
- response_object = StaticEngine(
- url,
- proxy,
- stealthy_headers,
- follow_redirects,
- timeout,
- retries,
- tuple(cookies.items()),
- adaptor_arguments=adaptor_arguments,
- ).put(**kwargs)
- return response_object
-
- @classmethod
- def delete(
- cls,
- url: str,
- follow_redirects: bool = True,
- timeout: Optional[Union[int, float]] = 10,
- stealthy_headers: bool = True,
- proxy: Optional[str] = None,
- retries: Optional[int] = 3,
- cookies: Optional[Dict] = None,
- custom_config: Dict = None,
- **kwargs: Dict,
- ) -> Response:
- """Make basic HTTP DELETE request for you but with some added flavors.
-
- :param url: Target url
- :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
- :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
- :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
- create a referer header as if this request came from Google's search of this URL's domain.
- :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
- :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
- :param cookies: Set cookies for the next request.
- :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
- :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
- """
- if not custom_config:
- custom_config = {}
- elif not isinstance(custom_config, dict):
- ValueError(
- f"The custom parser config must be of type dictionary, got {cls.__class__}"
- )
-
- adaptor_arguments = tuple(
- {**cls._generate_parser_arguments(), **custom_config}.items()
- )
-
- if not cookies:
- cookies = {}
- elif not isinstance(cookies, dict):
- ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
- response_object = StaticEngine(
- url,
- proxy,
- stealthy_headers,
- follow_redirects,
- timeout,
- retries,
- tuple(cookies.items()),
- adaptor_arguments=adaptor_arguments,
- ).delete(**kwargs)
- return response_object
-
-
-class AsyncFetcher(Fetcher):
- @classmethod
- async def get(
- cls,
- url: str,
- follow_redirects: bool = True,
- timeout: Optional[Union[int, float]] = 10,
- stealthy_headers: bool = True,
- proxy: Optional[str] = None,
- retries: Optional[int] = 3,
- cookies: Optional[Dict] = None,
- custom_config: Dict = None,
- **kwargs: Dict,
- ) -> Response:
- """Make basic HTTP GET request for you but with some added flavors.
-
- :param url: Target url.
- :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
- :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
- :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
- create a referer header as if this request had came from Google's search of this URL's domain.
- :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
- :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
- :param cookies: Set cookies for the next request.
- :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
- :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
- """
- if not custom_config:
- custom_config = {}
- elif not isinstance(custom_config, dict):
- ValueError(
- f"The custom parser config must be of type dictionary, got {cls.__class__}"
- )
-
- adaptor_arguments = tuple(
- {**cls._generate_parser_arguments(), **custom_config}.items()
- )
-
- if not cookies:
- cookies = {}
- elif not isinstance(cookies, dict):
- ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
- response_object = await StaticEngine(
- url,
- proxy,
- stealthy_headers,
- follow_redirects,
- timeout,
- retries=retries,
- cookies=tuple(cookies.items()),
- adaptor_arguments=adaptor_arguments,
- ).async_get(**kwargs)
- return response_object
-
- @classmethod
- async def post(
- cls,
- url: str,
- follow_redirects: bool = True,
- timeout: Optional[Union[int, float]] = 10,
- stealthy_headers: bool = True,
- proxy: Optional[str] = None,
- retries: Optional[int] = 3,
- cookies: Optional[Dict] = None,
- custom_config: Dict = None,
- **kwargs: Dict,
- ) -> Response:
- """Make basic HTTP POST request for you but with some added flavors.
-
- :param url: Target url.
- :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
- :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
- :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
- create a referer header as if this request came from Google's search of this URL's domain.
- :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
- :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
- :param cookies: Set cookies for the next request.
- :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
- :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
- """
- if not custom_config:
- custom_config = {}
- elif not isinstance(custom_config, dict):
- ValueError(
- f"The custom parser config must be of type dictionary, got {cls.__class__}"
- )
-
- adaptor_arguments = tuple(
- {**cls._generate_parser_arguments(), **custom_config}.items()
- )
-
- if not cookies:
- cookies = {}
- elif not isinstance(cookies, dict):
- ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
-
- response_object = await StaticEngine(
- url,
- proxy,
- stealthy_headers,
- follow_redirects,
- timeout,
- retries=retries,
- cookies=tuple(cookies.items()),
- adaptor_arguments=adaptor_arguments,
- ).async_post(**kwargs)
- return response_object
-
- @classmethod
- async def put(
- cls,
- url: str,
- follow_redirects: bool = True,
- timeout: Optional[Union[int, float]] = 10,
- stealthy_headers: bool = True,
- proxy: Optional[str] = None,
- retries: Optional[int] = 3,
- cookies: Optional[Dict] = None,
- custom_config: Dict = None,
- **kwargs: Dict,
- ) -> Response:
- """Make basic HTTP PUT request for you but with some added flavors.
-
- :param url: Target url
- :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
- :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
- :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
- create a referer header as if this request came from Google's search of this URL's domain.
- :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
- :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
- :param cookies: Set cookies for the next request.
- :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
- :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
- """
- if not custom_config:
- custom_config = {}
- elif not isinstance(custom_config, dict):
- ValueError(
- f"The custom parser config must be of type dictionary, got {cls.__class__}"
- )
-
- adaptor_arguments = tuple(
- {**cls._generate_parser_arguments(), **custom_config}.items()
- )
-
- if not cookies:
- cookies = {}
- elif not isinstance(cookies, dict):
- ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+class Fetcher(BaseFetcher):
+ """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
- response_object = await StaticEngine(
- url,
- proxy,
- stealthy_headers,
- follow_redirects,
- timeout,
- retries=retries,
- cookies=tuple(cookies.items()),
- adaptor_arguments=adaptor_arguments,
- ).async_put(**kwargs)
- return response_object
+ get = __FetcherClientInstance__.get
+ post = __FetcherClientInstance__.post
+ put = __FetcherClientInstance__.put
+ delete = __FetcherClientInstance__.delete
- @classmethod
- async def delete(
- cls,
- url: str,
- follow_redirects: bool = True,
- timeout: Optional[Union[int, float]] = 10,
- stealthy_headers: bool = True,
- proxy: Optional[str] = None,
- retries: Optional[int] = 3,
- cookies: Optional[Dict] = None,
- custom_config: Dict = None,
- **kwargs: Dict,
- ) -> Response:
- """Make basic HTTP DELETE request for you but with some added flavors.
-
- :param url: Target url
- :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
- :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
- :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
- create a referer header as if this request came from Google's search of this URL's domain.
- :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
- :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
- :param cookies: Set cookies for the next request.
- :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
- :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
- :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
- """
- if not custom_config:
- custom_config = {}
- elif not isinstance(custom_config, dict):
- ValueError(
- f"The custom parser config must be of type dictionary, got {cls.__class__}"
- )
-
- adaptor_arguments = tuple(
- {**cls._generate_parser_arguments(), **custom_config}.items()
- )
- if not cookies:
- cookies = {}
- elif not isinstance(cookies, dict):
- ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
+class AsyncFetcher(BaseFetcher):
+ """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
- response_object = await StaticEngine(
- url,
- proxy,
- stealthy_headers,
- follow_redirects,
- timeout,
- retries=retries,
- cookies=tuple(cookies.items()),
- adaptor_arguments=adaptor_arguments,
- ).async_delete(**kwargs)
- return response_object
+ get = _AsyncFetcherClient.get
+ post = _AsyncFetcherClient.post
+ put = _AsyncFetcherClient.put
+ delete = _AsyncFetcherClient.delete
class StealthyFetcher(BaseFetcher):
diff --git a/setup.py b/setup.py
index 28e1d54..ec5fe67 100644
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,6 @@ setup(
"Programming Language :: Python :: Implementation :: CPython",
"Typing :: Typed",
],
- # Instead of using requirements file to dodge possible errors from tox?
install_requires=[
"lxml>=5.0",
"cssselect>=1.2",
@@ -56,7 +55,7 @@ setup(
"click",
"orjson>=3",
"tldextract",
- "httpx[brotli,zstd, socks]",
+ "curl_cffi>=0.11.1",
"playwright>=1.49.1",
"rebrowser-playwright>=1.49.1",
"camoufox[geoip]>=0.4.11",