Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

PlaywrightEngine - Validate CDP URLs in all cases

Karim shoair6cf506cb22aa
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index a62a691..b08195c 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -96,6 +96,9 @@ class PlaywrightEngine:
# 'token': ''
}
cdp_url = construct_cdp_url(cdp_url, config)
+ else:
+ # To validate it
+ cdp_url = construct_cdp_url(cdp_url)
return cdp_url
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index 2af7b86..e03cfd7 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -6,7 +6,7 @@ import os
import logging
from urllib.parse import urlparse, urlencode
-from scrapling.core._types import Union, Dict
+from scrapling.core._types import Union, Dict, Optional
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
from playwright.sync_api import Route
@@ -24,7 +24,7 @@ def intercept_route(route: Route) -> Union[Route, None]:
return route.continue_()
-def construct_cdp_url(cdp_url: str, query_params: Dict) -> str:
+def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
"""Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
:param cdp_url: The target URL.