🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
PlaywrightEngine - Validate CDP URLs in all cases
Karim shoair6cf506cb22aa
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex a62a691..b08195c 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -96,6 +96,9 @@ class PlaywrightEngine:# 'token': ''}cdp_url = construct_cdp_url(cdp_url, config)+ else:+ # To validate it+ cdp_url = construct_cdp_url(cdp_url)return cdp_urldiff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex 2af7b86..e03cfd7 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -6,7 +6,7 @@ import osimport loggingfrom urllib.parse import urlparse, urlencode-from scrapling.core._types import Union, Dict+from scrapling.core._types import Union, Dict, Optionalfrom scrapling.engines.constants import DEFAULT_DISABLED_RESOURCESfrom playwright.sync_api import Route@@ -24,7 +24,7 @@ def intercept_route(route: Route) -> Union[Route, None]:return route.continue_()-def construct_cdp_url(cdp_url: str, query_params: Dict) -> str:+def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:"""Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists:param cdp_url: The target URL.