🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Get page content after JS Execution
Karim shoair43045e3eb2fd
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.pyindex f56ad69..61731fc 100644--- a/scrapling/engines/camo.py+++ b/scrapling/engines/camo.py@@ -87,9 +87,7 @@ class CamoufoxEngine:if self.extra_headers:page.set_extra_http_headers(self.extra_headers)- res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)- page.wait_for_load_state(state="load")- page.wait_for_load_state(state="domcontentloaded")+ res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")if self.network_idle:page.wait_for_load_state('networkidle')diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex 4f453fe..72f571a 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -199,9 +199,7 @@ class PlaywrightEngine:page.add_init_script(path=js_bypass_path('screen_props.js'))page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))- res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)- page.wait_for_load_state(state="load")- page.wait_for_load_state(state="domcontentloaded")+ res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")if self.network_idle:page.wait_for_load_state('networkidle')