Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

PlaywrightFetcher - The option to drop unnecessary resources requests for speed boost

Karim shoair711de8110637
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index 4d628c3..4ce0600 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -7,6 +7,7 @@ from scrapling.engines.toolbelt import (
Response,
do_nothing,
js_bypass_path,
+ intercept_route,
generate_headers,
check_type_validity,
construct_websocket_url,
@@ -17,7 +18,7 @@ from scrapling.engines.toolbelt import (
class PlaywrightEngine:
def __init__(
self, headless: Union[bool, str] = True,
- disable_resources: Optional[List] = None,
+ disable_resources: Optional[bool] = False,
useragent: Optional[str] = None,
network_idle: Optional[bool] = False,
timeout: Optional[float] = 30000,
@@ -134,6 +135,9 @@ class PlaywrightEngine:
page = context.new_page()
page.set_default_navigation_timeout(self.timeout)
page.set_default_timeout(self.timeout)
+ if self.disable_resources:
+ page.route("**/*", intercept_route)
+
if self.stealth:
# Basic bypasses nothing fancy as I'm still working on it
# But with adding these bypasses to the above config, it bypasses many online tests like