Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Adding the proxy support to browser-based Fetchers

Karim shoair2abb70218f46
diff --git a/README.md b/README.md
index 33ebbf9..d19981e 100644
--- a/README.md
+++ b/README.md
@@ -257,6 +257,7 @@ True
| network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ |
| timeout | The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000. | ✔️ |
| wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ |
+| proxy | The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ |
| wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
</details>
@@ -299,6 +300,7 @@ Add that to a lot of controlling/hiding options as you will see in the arguments
| wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |
| google_search | Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name. | ✔️ |
| extra_headers | A dictionary of extra headers to add to the request. The referer set by the `google_search` argument takes priority over the referer set here if used together. | ✔️ |
+| proxy | The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ |
| hide_canvas | Add random noise to canvas operations to prevent fingerprinting. | ✔️ |
| disable_webgl | Disables WebGL and WebGL 2.0 support entirely. | ✔️ |
| stealth | Enables stealth mode, always check the documentation to see what stealth mode does currently. | ✔️ |
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 3677531..6a26495 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -7,6 +7,7 @@ from scrapling.engines.toolbelt import (
get_os_name,
intercept_route,
check_type_validity,
+ construct_proxy_dict,
generate_convincing_referer,
)
@@ -18,7 +19,8 @@ class CamoufoxEngine:
self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,
timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,
- wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, adaptor_arguments: Dict = None
+ wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,
+ proxy: Optional[Union[str, Dict[str, str]]] = None, adaptor_arguments: Dict = None
):
"""An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.
@@ -39,6 +41,7 @@ class CamoufoxEngine:
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
"""
self.headless = headless
@@ -49,6 +52,7 @@ class CamoufoxEngine:
self.network_idle = bool(network_idle)
self.google_search = bool(google_search)
self.extra_headers = extra_headers or {}
+ self.proxy = construct_proxy_dict(proxy)
self.addons = addons or []
self.humanize = humanize
self.timeout = check_type_validity(timeout, [int, float], 30000)
@@ -76,6 +80,7 @@ class CamoufoxEngine:
allow_webgl=self.allow_webgl,
addons=self.addons,
humanize=self.humanize,
+ proxy=self.proxy,
i_know_what_im_doing=True, # To turn warnings off with user configurations
) as browser:
page = browser.new_page()
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index 2d6ebf2..b97bcac 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -9,8 +9,9 @@ from scrapling.engines.toolbelt import (
js_bypass_path,
intercept_route,
generate_headers,
- check_type_validity,
construct_cdp_url,
+ check_type_validity,
+ construct_proxy_dict,
generate_convincing_referer,
)
@@ -33,6 +34,7 @@ class PlaywrightEngine:
nstbrowser_config: Optional[Dict] = None,
google_search: Optional[bool] = True,
extra_headers: Optional[Dict[str, str]] = None,
+ proxy: Optional[Union[str, Dict[str, str]]] = None,
adaptor_arguments: Dict = None
):
"""An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation.
@@ -54,6 +56,7 @@ class PlaywrightEngine:
:param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class.
"""
@@ -65,6 +68,7 @@ class PlaywrightEngine:
self.disable_webgl = bool(disable_webgl)
self.google_search = bool(google_search)
self.extra_headers = extra_headers or {}
+ self.proxy = construct_proxy_dict(proxy)
self.cdp_url = cdp_url
self.useragent = useragent
self.timeout = check_type_validity(timeout, [int, float], 30000)
@@ -151,6 +155,7 @@ class PlaywrightEngine:
locale='en-US',
is_mobile=False,
has_touch=False,
+ proxy=self.proxy,
color_scheme='dark', # Bypasses the 'prefersLightColor' check in creepjs
user_agent=useragent,
device_scale_factor=2,
diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py
index 08b559b..ac3e03d 100644
--- a/scrapling/engines/toolbelt/__init__.py
+++ b/scrapling/engines/toolbelt/__init__.py
@@ -15,4 +15,5 @@ from .navigation import (
js_bypass_path,
intercept_route,
construct_cdp_url,
+ construct_proxy_dict,
)
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index cf73a39..363f233 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -25,6 +25,40 @@ def intercept_route(route: Route) -> Union[Route, None]:
return route.continue_()
+def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict, None]:
+ """Validate a proxy and return it in the acceptable format for Playwright
+ Reference: https://playwright.dev/python/docs/network#http-proxy
+
+ :param proxy_string: A string or a dictionary representation of the proxy.
+ :return:
+ """
+ if proxy_string:
+ if isinstance(proxy_string, str):
+ proxy = urlparse(proxy_string)
+ try:
+ return {
+ 'server': f'{proxy.scheme}://{proxy.hostname}:{proxy.port}',
+ 'username': proxy.username or '',
+ 'password': proxy.password or '',
+ }
+ except ValueError:
+ # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
+ raise TypeError(f'The proxy argument\'s string is in invalid format!')
+
+ elif isinstance(proxy_string, dict):
+ valid_keys = ('server', 'username', 'password', )
+ if all(key in valid_keys for key in proxy_string.keys()) and not any(key not in valid_keys for key in proxy_string.keys()):
+ return proxy_string
+ else:
+ raise TypeError(f'A proxy dictionary must have only these keys: {valid_keys}')
+
+ else:
+ raise TypeError(f'Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!')
+
+ # The default value for proxy in Playwright's source is `None`
+ return None
+
+
def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
"""Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py
index 65a8901..b0f1d60 100644
--- a/scrapling/fetchers.py
+++ b/scrapling/fetchers.py
@@ -72,7 +72,7 @@ class StealthyFetcher(BaseFetcher):
self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,
block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,
timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,
- wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None
+ wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None,
) -> Response:
"""
Opens up a browser and do your request based on your chosen options below.
@@ -94,23 +94,25 @@ class StealthyFetcher(BaseFetcher):
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
"""
engine = CamoufoxEngine(
+ proxy=proxy,
+ addons=addons,
timeout=timeout,
headless=headless,
- page_action=page_action,
- block_images=block_images,
- block_webrtc=block_webrtc,
- addons=addons,
humanize=humanize,
allow_webgl=allow_webgl,
- disable_resources=disable_resources,
+ page_action=page_action,
network_idle=network_idle,
+ block_images=block_images,
+ block_webrtc=block_webrtc,
wait_selector=wait_selector,
- wait_selector_state=wait_selector_state,
google_search=google_search,
extra_headers=extra_headers,
+ disable_resources=disable_resources,
+ wait_selector_state=wait_selector_state,
adaptor_arguments=self.adaptor_arguments,
)
return engine.fetch(url)
@@ -136,6 +138,7 @@ class PlayWrightFetcher(BaseFetcher):
useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,
page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',
hide_canvas: bool = True, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,
+ proxy: Optional[Union[str, Dict[str, str]]] = None,
stealth: bool = False,
cdp_url: Optional[str] = None,
nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,
@@ -157,12 +160,14 @@ class PlayWrightFetcher(BaseFetcher):
:param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.
:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.
:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._
+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.
:param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.
:param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.
:return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course.
"""
engine = PlaywrightEngine(
+ proxy=proxy,
timeout=timeout,
stealth=stealth,
cdp_url=cdp_url,