Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: improve checking for valid proxy and valid CDP URL

Karim shoair825896c6a843
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index dd4c40c..ab91174 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -65,16 +65,24 @@ def construct_proxy_dict(
"""
if isinstance(proxy_string, str):
proxy = urlparse(proxy_string)
+ if (
+ proxy.scheme not in ("http", "https", "socks4", "socks5")
+ or not proxy.hostname
+ ):
+ raise ValueError("Invalid proxy string!")
+
try:
result = {
- "server": f"{proxy.scheme}://{proxy.hostname}:{proxy.port}",
+ "server": f"{proxy.scheme}://{proxy.hostname}",
"username": proxy.username or "",
"password": proxy.password or "",
}
+ if proxy.port:
+ result["server"] += f":{proxy.port}"
return tuple(result.items()) if as_tuple else result
except ValueError:
# Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
- raise TypeError("The proxy argument's string is in invalid format!")
+ raise ValueError("The proxy argument's string is in invalid format!")
elif isinstance(proxy_string, dict):
try:
@@ -106,6 +114,13 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
if not parsed.netloc:
raise ValueError("Invalid hostname for the CDP URL")
+ try:
+ # Checking if the port is valid (if available)
+ _ = parsed.port
+ except ValueError:
+ # urlparse will raise `ValueError` if the port can't be casted to integer
+ raise ValueError("Invalid port for the CDP URL")
+
# Ensure the path starts with /
path = parsed.path
if not path.startswith("/"):