🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: improve checking for valid proxy and valid CDP URL
Karim shoair825896c6a843
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex dd4c40c..ab91174 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -65,16 +65,24 @@ def construct_proxy_dict("""if isinstance(proxy_string, str):proxy = urlparse(proxy_string)+ if (+ proxy.scheme not in ("http", "https", "socks4", "socks5")+ or not proxy.hostname+ ):+ raise ValueError("Invalid proxy string!")+try:result = {- "server": f"{proxy.scheme}://{proxy.hostname}:{proxy.port}",+ "server": f"{proxy.scheme}://{proxy.hostname}","username": proxy.username or "","password": proxy.password or "",}+ if proxy.port:+ result["server"] += f":{proxy.port}"return tuple(result.items()) if as_tuple else resultexcept ValueError:# Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...- raise TypeError("The proxy argument's string is in invalid format!")+ raise ValueError("The proxy argument's string is in invalid format!")elif isinstance(proxy_string, dict):try:@@ -106,6 +114,13 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:if not parsed.netloc:raise ValueError("Invalid hostname for the CDP URL")+ try:+ # Checking if the port is valid (if available)+ _ = parsed.port+ except ValueError:+ # urlparse will raise `ValueError` if the port can't be casted to integer+ raise ValueError("Invalid port for the CDP URL")+# Ensure the path starts with /path = parsed.pathif not path.startswith("/"):