🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Engines utils - Navigation functions
Karim shoair2ff0fbb0e554
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex aa2e0c2..67ea49e 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -10,7 +10,7 @@ from scrapling.engines.toolbelt import (intercept_route,generate_headers,check_type_validity,- construct_websocket_url,+ construct_cdp_url,generate_convincing_referer,)@@ -95,7 +95,7 @@ class PlaywrightEngine:'config': json.dumps(query),# 'token': ''}- cdp_url = construct_websocket_url(cdp_url, config)+ cdp_url = construct_cdp_url(cdp_url, config)return cdp_urldiff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.pyindex 375e304..7e8368a 100644--- a/scrapling/engines/toolbelt/__init__.py+++ b/scrapling/engines/toolbelt/__init__.py@@ -15,5 +15,5 @@ from .custom import (from .navigation import (js_bypass_path,intercept_route,- construct_websocket_url,+ construct_cdp_url,)diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex 5a8192b..111b17c 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -8,27 +8,39 @@ from urllib.parse import urlparse, urlencodefrom playwright.sync_api import Routefrom scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES+from scrapling.core._types import Union, Dict-def intercept_route(route: Route):+def intercept_route(route: Route) -> Union[Route, None]:+ """This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`++ :param route: PlayWright `Route` object of the current page+ :return: PlayWright `Route` object+ """if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:logging.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')return route.abort()return route.continue_()-def construct_websocket_url(base_url, query_params):- # Validate the base URL structure+def construct_cdp_url(cdp_url: str, query_params: Dict) -> str:+ """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists++ :param cdp_url: The target URL.+ :param query_params: A dictionary of the parameters to add.+ :return: The new CDP URL.+ """try:- parsed = urlparse(base_url)+ # Validate the base URL structure+ parsed = urlparse(cdp_url)# Check schemeif parsed.scheme not in ('ws', 'wss'):- raise ValueError("URL must use 'ws://' or 'wss://' scheme")+ raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")# Validate hostname and portif not parsed.netloc:- raise ValueError("Invalid hostname")+ raise ValueError("Invalid hostname for the CDP URL")# Ensure path starts with /path = parsed.path@@ -46,9 +58,14 @@ def construct_websocket_url(base_url, query_params):return validated_baseexcept Exception as e:- raise ValueError(f"Invalid WebSocket URL: {str(e)}")+ raise ValueError(f"Invalid CDP URL: {str(e)}")++def js_bypass_path(filename: str) -> str:+ """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it-def js_bypass_path(filename):+ :param filename: The base filename of the JS file.+ :return: The full path of the JS file.+ """current_directory = os.path.dirname(__file__)return os.path.join(current_directory, 'bypasses', filename)