🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(navigation tools): Improvements
Karim shoair9bfc900140e6
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex 167d4d6..1b00f24 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -3,16 +3,23 @@ Functions related to files and URLs"""import os+import msgspecfrom urllib.parse import urlencode, urlparsefrom playwright.async_api import Route as async_Routefrom playwright.sync_api import Route-from scrapling.core._types import Dict, Optional, Union+from scrapling.core._types import Dict, Optional, Union, Tuplefrom scrapling.core.utils import log, lru_cachefrom scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES+class ProxyDict(msgspec.Struct):+ server: str+ username: str = ""+ password: str = ""++def intercept_route(route: Route):"""This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`@@ -43,47 +50,36 @@ async def async_intercept_route(route: async_Route):await route.continue_()-def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict, None]:+def construct_proxy_dict(+ proxy_string: Union[str, Dict[str, str]], as_tuple=False+) -> Union[Dict, Tuple, None]:"""Validate a proxy and return it in the acceptable format for PlaywrightReference: https://playwright.dev/python/docs/network#http-proxy:param proxy_string: A string or a dictionary representation of the proxy.+ :param as_tuple: Return the proxy dictionary as tuple to be cachable:return:"""- if proxy_string:- if isinstance(proxy_string, str):- proxy = urlparse(proxy_string)- try:- return {- "server": f"{proxy.scheme}://{proxy.hostname}:{proxy.port}",- "username": proxy.username or "",- "password": proxy.password or "",- }- except ValueError:- # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...- raise TypeError("The proxy argument's string is in invalid format!")-- elif isinstance(proxy_string, dict):- valid_keys = (- "server",- "username",- "password",- )- if all(key in valid_keys for key in proxy_string.keys()) and not any(- key not in valid_keys for key in proxy_string.keys()- ):- return proxy_string- else:- raise TypeError(- f"A proxy dictionary must have only these keys: {valid_keys}"- )-- else:- raise TypeError(- f"Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!"- )-- # The default value for proxy in Playwright's source is `None`+ if isinstance(proxy_string, str):+ proxy = urlparse(proxy_string)+ try:+ result = {+ "server": f"{proxy.scheme}://{proxy.hostname}:{proxy.port}",+ "username": proxy.username or "",+ "password": proxy.password or "",+ }+ return tuple(result.items()) if as_tuple else result+ except ValueError:+ # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...+ raise TypeError("The proxy argument's string is in invalid format!")++ elif isinstance(proxy_string, dict):+ try:+ validated = msgspec.convert(proxy_string, ProxyDict)+ return tuple(validated.__dict__.items()) if as_tuple else validated.__dict__+ except msgspec.ValidationError as e:+ raise TypeError(f"Invalid proxy dictionary: {e}")+return None