Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Adding a function to intercept and drop unwanted requests

Karim shoairc8c494e73f09
diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.py
index 3aa0aed..375e304 100644
--- a/scrapling/engines/toolbelt/__init__.py
+++ b/scrapling/engines/toolbelt/__init__.py
@@ -14,5 +14,6 @@ from .custom import (
)
from .navigation import (
js_bypass_path,
+ intercept_route,
construct_websocket_url,
)
diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.py
index bf99cb2..5a8192b 100644
--- a/scrapling/engines/toolbelt/navigation.py
+++ b/scrapling/engines/toolbelt/navigation.py
@@ -3,7 +3,18 @@ Functions related to files and URLs
"""
import os
+import logging
from urllib.parse import urlparse, urlencode
+from playwright.sync_api import Route
+
+from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES
+
+
+def intercept_route(route: Route):
+ if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
+ logging.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
+ return route.abort()
+ return route.continue_()
def construct_websocket_url(base_url, query_params):