🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Adding a function to intercept and drop unwanted requests
Karim shoairc8c494e73f09
diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.pyindex 3aa0aed..375e304 100644--- a/scrapling/engines/toolbelt/__init__.py+++ b/scrapling/engines/toolbelt/__init__.py@@ -14,5 +14,6 @@ from .custom import ()from .navigation import (js_bypass_path,+ intercept_route,construct_websocket_url,)diff --git a/scrapling/engines/toolbelt/navigation.py b/scrapling/engines/toolbelt/navigation.pyindex bf99cb2..5a8192b 100644--- a/scrapling/engines/toolbelt/navigation.py+++ b/scrapling/engines/toolbelt/navigation.py@@ -3,7 +3,18 @@ Functions related to files and URLs"""import os+import loggingfrom urllib.parse import urlparse, urlencode+from playwright.sync_api import Route++from scrapling.engines.constants import DEFAULT_DISABLED_RESOURCES+++def intercept_route(route: Route):+ if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:+ logging.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')+ return route.abort()+ return route.continue_()def construct_websocket_url(base_url, query_params):