🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Adding the proxy support to browser-based Fetchers
Karim shoair2abb70218f46
diff --git a/README.md b/README.mdindex 33ebbf9..d19981e 100644--- a/README.md+++ b/README.md@@ -257,6 +257,7 @@ True| network_idle | Wait for the page until there are no network connections for at least 500 ms. | ✔️ || timeout | The timeout in milliseconds that is used in all operations and waits through the page. The default is 30000. | ✔️ || wait_selector | Wait for a specific css selector to be in a specific state. | ✔️ |+| proxy | The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ || wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ |</details>@@ -299,6 +300,7 @@ Add that to a lot of controlling/hiding options as you will see in the arguments| wait_selector_state | The state to wait for the selector given with `wait_selector`. _Default state is `attached`._ | ✔️ || google_search | Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name. | ✔️ || extra_headers | A dictionary of extra headers to add to the request. The referer set by the `google_search` argument takes priority over the referer set here if used together. | ✔️ |+| proxy | The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. | ✔️ || hide_canvas | Add random noise to canvas operations to prevent fingerprinting. | ✔️ || disable_webgl | Disables WebGL and WebGL 2.0 support entirely. | ✔️ || stealth | Enables stealth mode, always check the documentation to see what stealth mode does currently. | ✔️ |diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.pyindex 3677531..6a26495 100644--- a/scrapling/engines/camo.py+++ b/scrapling/engines/camo.py@@ -7,6 +7,7 @@ from scrapling.engines.toolbelt import (get_os_name,intercept_route,check_type_validity,+ construct_proxy_dict,generate_convincing_referer,)@@ -18,7 +19,8 @@ class CamoufoxEngine:self, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, humanize: Optional[Union[bool, float]] = True,timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, addons: Optional[List[str]] = None,- wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, adaptor_arguments: Dict = None+ wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None,+ proxy: Optional[Union[str, Dict[str, str]]] = None, adaptor_arguments: Dict = None):"""An engine that utilizes Camoufox library, check the `StealthyFetcher` class for more documentation.@@ -39,6 +41,7 @@ class CamoufoxEngine::param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class."""self.headless = headless@@ -49,6 +52,7 @@ class CamoufoxEngine:self.network_idle = bool(network_idle)self.google_search = bool(google_search)self.extra_headers = extra_headers or {}+ self.proxy = construct_proxy_dict(proxy)self.addons = addons or []self.humanize = humanizeself.timeout = check_type_validity(timeout, [int, float], 30000)@@ -76,6 +80,7 @@ class CamoufoxEngine:allow_webgl=self.allow_webgl,addons=self.addons,humanize=self.humanize,+ proxy=self.proxy,i_know_what_im_doing=True, # To turn warnings off with user configurations) as browser:page = browser.new_page()diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex 2d6ebf2..b97bcac 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -9,8 +9,9 @@ from scrapling.engines.toolbelt import (js_bypass_path,intercept_route,generate_headers,- check_type_validity,construct_cdp_url,+ check_type_validity,+ construct_proxy_dict,generate_convincing_referer,)@@ -33,6 +34,7 @@ class PlaywrightEngine:nstbrowser_config: Optional[Dict] = None,google_search: Optional[bool] = True,extra_headers: Optional[Dict[str, str]] = None,+ proxy: Optional[Union[str, Dict[str, str]]] = None,adaptor_arguments: Dict = None):"""An engine that utilizes PlayWright library, check the `PlayWrightFetcher` class for more documentation.@@ -54,6 +56,7 @@ class PlaywrightEngine::param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.:param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.:param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class."""@@ -65,6 +68,7 @@ class PlaywrightEngine:self.disable_webgl = bool(disable_webgl)self.google_search = bool(google_search)self.extra_headers = extra_headers or {}+ self.proxy = construct_proxy_dict(proxy)self.cdp_url = cdp_urlself.useragent = useragentself.timeout = check_type_validity(timeout, [int, float], 30000)@@ -151,6 +155,7 @@ class PlaywrightEngine:locale='en-US',is_mobile=False,has_touch=False,+ proxy=self.proxy,color_scheme='dark', # Bypasses the 'prefersLightColor' check in creepjsuser_agent=useragent,device_scale_factor=2,diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.pyindex 08b559b..ac3e03d 100644--- a/scrapling/engines/toolbelt/__init__.py+++ b/scrapling/engines/toolbelt/__init__.py@@ -15,4 +15,5 @@ from .navigation import (js_bypass_path,intercept_route,construct_cdp_url,+ construct_proxy_dict,)diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex cf73a39..363f233 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -25,6 +25,40 @@ def intercept_route(route: Route) -> Union[Route, None]:return route.continue_()+def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict, None]:+ """Validate a proxy and return it in the acceptable format for Playwright+ Reference: https://playwright.dev/python/docs/network#http-proxy++ :param proxy_string: A string or a dictionary representation of the proxy.+ :return:+ """+ if proxy_string:+ if isinstance(proxy_string, str):+ proxy = urlparse(proxy_string)+ try:+ return {+ 'server': f'{proxy.scheme}://{proxy.hostname}:{proxy.port}',+ 'username': proxy.username or '',+ 'password': proxy.password or '',+ }+ except ValueError:+ # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...+ raise TypeError(f'The proxy argument\'s string is in invalid format!')++ elif isinstance(proxy_string, dict):+ valid_keys = ('server', 'username', 'password', )+ if all(key in valid_keys for key in proxy_string.keys()) and not any(key not in valid_keys for key in proxy_string.keys()):+ return proxy_string+ else:+ raise TypeError(f'A proxy dictionary must have only these keys: {valid_keys}')++ else:+ raise TypeError(f'Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!')++ # The default value for proxy in Playwright's source is `None`+ return None++def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:"""Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if existsdiff --git a/scrapling/fetchers.py b/scrapling/fetchers.pyindex 65a8901..b0f1d60 100644--- a/scrapling/fetchers.py+++ b/scrapling/fetchers.py@@ -72,7 +72,7 @@ class StealthyFetcher(BaseFetcher):self, url: str, headless: Optional[Union[bool, Literal['virtual']]] = True, block_images: Optional[bool] = False, disable_resources: Optional[bool] = False,block_webrtc: Optional[bool] = False, allow_webgl: Optional[bool] = False, network_idle: Optional[bool] = False, addons: Optional[List[str]] = None,timeout: Optional[float] = 30000, page_action: Callable = do_nothing, wait_selector: Optional[str] = None, humanize: Optional[Union[bool, float]] = True,- wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None+ wait_selector_state: str = 'attached', google_search: Optional[bool] = True, extra_headers: Optional[Dict[str, str]] = None, proxy: Optional[Union[str, Dict[str, str]]] = None,) -> Response:"""Opens up a browser and do your request based on your chosen options below.@@ -94,23 +94,25 @@ class StealthyFetcher(BaseFetcher)::param wait_selector_state: The state to wait for the selector given with `wait_selector`. Default state is `attached`.:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.:return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course."""engine = CamoufoxEngine(+ proxy=proxy,+ addons=addons,timeout=timeout,headless=headless,- page_action=page_action,- block_images=block_images,- block_webrtc=block_webrtc,- addons=addons,humanize=humanize,allow_webgl=allow_webgl,- disable_resources=disable_resources,+ page_action=page_action,network_idle=network_idle,+ block_images=block_images,+ block_webrtc=block_webrtc,wait_selector=wait_selector,- wait_selector_state=wait_selector_state,google_search=google_search,extra_headers=extra_headers,+ disable_resources=disable_resources,+ wait_selector_state=wait_selector_state,adaptor_arguments=self.adaptor_arguments,)return engine.fetch(url)@@ -136,6 +138,7 @@ class PlayWrightFetcher(BaseFetcher):useragent: Optional[str] = None, network_idle: Optional[bool] = False, timeout: Optional[float] = 30000,page_action: Callable = do_nothing, wait_selector: Optional[str] = None, wait_selector_state: Optional[str] = 'attached',hide_canvas: bool = True, disable_webgl: bool = False, extra_headers: Optional[Dict[str, str]] = None, google_search: Optional[bool] = True,+ proxy: Optional[Union[str, Dict[str, str]]] = None,stealth: bool = False,cdp_url: Optional[str] = None,nstbrowser_mode: bool = False, nstbrowser_config: Optional[Dict] = None,@@ -157,12 +160,14 @@ class PlayWrightFetcher(BaseFetcher)::param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.:param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search for this website's domain name.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.:param nstbrowser_mode: Enables NSTBrowser mode, it have to be used with `cdp_url` argument or it will get completely ignored.:param nstbrowser_config: The config you want to send with requests to the NSTBrowser. If left empty, Scrapling defaults to an optimized NSTBrowser's docker browserless config.:return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course."""engine = PlaywrightEngine(+ proxy=proxy,timeout=timeout,stealth=stealth,cdp_url=cdp_url,