Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Correctly get page content after JS Execution

Karim shoair239e2c73bfe4
diff --git a/scrapling/engines/camo.py b/scrapling/engines/camo.py
index 61731fc..6763b2b 100644
--- a/scrapling/engines/camo.py
+++ b/scrapling/engines/camo.py
@@ -87,7 +87,8 @@ class CamoufoxEngine:
if self.extra_headers:
page.set_extra_http_headers(self.extra_headers)
- res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
+ res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+ page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
@@ -105,7 +106,7 @@ class CamoufoxEngine:
response = Response(
url=res.url,
- text=res.text(),
+ text=page.content(),
content=res.body(),
status=res.status,
reason=res.status_text,
diff --git a/scrapling/engines/pw.py b/scrapling/engines/pw.py
index 72f571a..e3aa654 100644
--- a/scrapling/engines/pw.py
+++ b/scrapling/engines/pw.py
@@ -199,7 +199,8 @@ class PlaywrightEngine:
page.add_init_script(path=js_bypass_path('screen_props.js'))
page.add_init_script(path=js_bypass_path('playwright_fingerprint.js'))
- res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None, wait_until="domcontentloaded")
+ res = page.goto(url, referer=generate_convincing_referer(url) if self.google_search else None)
+ page.wait_for_load_state(state="domcontentloaded")
if self.network_idle:
page.wait_for_load_state('networkidle')
@@ -217,7 +218,7 @@ class PlaywrightEngine:
response = Response(
url=res.url,
- text=res.text(),
+ text=page.content(),
content=res.body(),
status=res.status,
reason=res.status_text,
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 9a1ca33..3688be2 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -28,10 +28,14 @@ class Response:
def adaptor(self) -> Union[Adaptor, None]:
"""Generate Adaptor instance from this response if possible, otherwise return None"""
automatch_domain = self.adaptor_arguments.pop('automatch_domain', None)
- if self.content:
- return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
- elif self.text:
+ if self.text:
+ # For playwright that will be the response after all JS executed
return Adaptor(text=self.text, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
+ elif self.content:
+ # For playwright, that's after all JS is loaded but not all of them executed, because playwright doesn't offer something like page.content()
+ # To get response Bytes after the load states
+ # Reference: https://playwright.dev/python/docs/api/class-page
+ return Adaptor(body=self.content, url=automatch_domain or self.url, encoding=self.encoding, **self.adaptor_arguments)
return None
def __repr__(self):