🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Correctly get page content after JS Execution
Karim shoair239e2c73bfe4
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.pyindex 61731fc..6763b2b 100644--- a/scrapling/engines/camo.py+++ b/scrapling/engines/camo.py@@ -87,7 +87,8 @@ class CamoufoxEngine:if self.extra_headers:page.set_extra_http_headers(self.extra_headers)- res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")+ res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)+ page.wait_for_load_state(state="domcontentloaded")if self.network_idle:page.wait_for_load_state('networkidle')@@ -105,7 +106,7 @@ class CamoufoxEngine:response = Response(url=res.url,- text=res.text(),+ text=page.content(),content=res.body(),status=res.status,reason=res.status_text,diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.pyindex 72f571a..e3aa654 100644--- a/scrapling/engines/pw.py+++ b/scrapling/engines/pw.py@@ -199,7 +199,8 @@ class PlaywrightEngine:page.add_init_script(path=js_bypass_path('screen_props.js'))page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))- res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")+ res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)+ page.wait_for_load_state(state="domcontentloaded")if self.network_idle:page.wait_for_load_state('networkidle')@@ -217,7 +218,7 @@ class PlaywrightEngine:response = Response(url=res.url,- text=res.text(),+ text=page.content(),content=res.body(),status=res.status,reason=res.status_text,diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 9a1ca33..3688be2 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -28,10 +28,14 @@ class Response:def adaptor(self) -> Union[Adaptor, None]:"""Generate Adaptor instance from this response if possible, otherwise return None"""automatch_domain = self.adaptor_arguments.pop('automatch_domain', None)- if self.content:- return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)- elif self.text:+ if self.text:+ # For playwright that will be the response after all JS executedreturn Adaptor(text=self.text, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)+ elif self.content:+ # For playwright, that's after all JS is loaded but not all of them executed, because playwright doesn't offer something like page.content()+ # To get response Bytes after the load states+ # Reference: https://playwright.dev/python/docs/api/class-page+ return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)return Nonedef __repr__(self):