Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Engines utils - Navigation functions

Karim shoair2ff0fbb0e554
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index aa2e0c2..67ea49e 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -10,7 +10,7 @@ from scrapling.engines.toolbelt import (
intercept_route,
generate_headers,
check_type_validity,
- construct_websocket_url,
+ construct_cdp_url,
generate_convincing_referer,
)
@@ -95,7 +95,7 @@ class PlaywrightEngine:
'config': json.dumps(query),
# 'token': ''
}
- cdp_url = construct_websocket_url(cdp_url, config)
+ cdp_url = construct_cdp_url(cdp_url, config)
return cdp_url
diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py
index 375e304..7e8368a 100644
--- a/scrapling/engines/toolbelt/__init__.py
+++ b/scrapling/engines/toolbelt/__init__.py
@@ -15,5 +15,5 @@ from .custom import (
from .navigation import (
js_bypass_path,
intercept_route,
- construct_websocket_url,
+ construct_cdp_url,
)
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index 5a8192b..111b17c 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -8,27 +8,39 @@ from urllib.parse import urlparse, urlencode
from playwright.sync_api import Route
from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
+from scrapling.core._types import Union, Dict
-def intercept_route(route: Route):
+def intercept_route(route: Route) -> Union[Route, None]:
+ """This is just a route handler but it drops requests that its type falls in `DEFAULT_DISABLED_RESOURCES`
+
+ :param route: PlayWright `Route` object of the current page
+ :return: PlayWright `Route` object
+ """
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
logging.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
return route.abort()
return route.continue_()
-def construct_websocket_url(base_url, query_params):
- # Validate the base URL structure
+def construct_cdp_url(cdp_url: str, query_params: Dict) -> str:
+ """Takes a CDP URL, reconstruct it to check it's valid, then adds encoded parameters if exists
+
+ :param cdp_url: The target URL.
+ :param query_params: A dictionary of the parameters to add.
+ :return: The new CDP URL.
+ """
try:
- parsed = urlparse(base_url)
+ # Validate the base URL structure
+ parsed = urlparse(cdp_url)
# Check scheme
if parsed.scheme not in ('ws', 'wss'):
- raise ValueError("URL must use 'ws://' or 'wss://' scheme")
+ raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
if not parsed.netloc:
- raise ValueError("Invalid hostname")
+ raise ValueError("Invalid hostname for the CDP URL")
# Ensure path starts with /
path = parsed.path
@@ -46,9 +58,14 @@ def construct_websocket_url(base_url, query_params):
return validated_base
except Exception as e:
- raise ValueError(f"Invalid WebSocket URL: {str(e)}")
+ raise ValueError(f"Invalid CDP URL: {str(e)}")
+
+def js_bypass_path(filename: str) -> str:
+ """Takes the base filename of JS file inside the `bypasses` folder then return the full path of it
-def js_bypass_path(filename):
+ :param filename: The base filename of the JS file.
+ :return: The full path of the JS file.
+ """
current_directory = os.path.dirname(__file__)
return os.path.join(current_directory, 'bypasses', filename)