Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: make stealth init scripts available to page actions

mehmet turac9a6d3a81baa8
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index c9d390c..14b63ca 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -146,6 +146,12 @@ class SyncSession:
if network_idle:
self._wait_for_networkidle(page)
+ def _replay_init_script_for_page_action(self, page: Page) -> None:
+ if self._config.init_script:
+ # Make init_script globals visible to the automation execution context.
+ with open(self._config.init_script, encoding="utf-8") as script_file:
+ page.evaluate(script_file.read())
+
@staticmethod
def _create_response_handler(
page_info: PageInfo[Page],
@@ -333,6 +339,12 @@ class AsyncSession:
if network_idle:
await self._wait_for_networkidle(page)
+ async def _replay_init_script_for_page_action(self, page: AsyncPage) -> None:
+ if self._config.init_script:
+ # Make init_script globals visible to the automation execution context.
+ with open(self._config.init_script, encoding="utf-8") as script_file:
+ await page.evaluate(script_file.read())
+
@staticmethod
def _create_response_handler(
page_info: PageInfo[AsyncPage],
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index efa195e..39ba525 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -256,6 +256,11 @@ class StealthySession(SyncSession, StealthySessionMixin):
self._wait_for_page_stability(page, params.load_dom, params.network_idle)
if params.page_action:
+ try:
+ self._replay_init_script_for_page_action(page)
+ except Exception as e: # pragma: no cover
+ log.error(f"Error executing init_script before page_action: {e}")
+
try:
_ = params.page_action(page)
except Exception as e: # pragma: no cover
@@ -532,6 +537,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
if params.page_action:
+ try:
+ await self._replay_init_script_for_page_action(page)
+ except Exception as e: # pragma: no cover
+ log.error(f"Error executing init_script before page_action: {e}")
+
try:
_ = await params.page_action(page)
except Exception as e: # pragma: no cover
diff --git a/tests/fetchers/async/test_stealth_session.py b/tests/fetchers/async/test_stealth_session.py
index db79002..b37d4a0 100644
--- a/tests/fetchers/async/test_stealth_session.py
+++ b/tests/fetchers/async/test_stealth_session.py
@@ -78,6 +78,21 @@ class TestAsyncStealthySession:
response = await session.fetch(urls["html"])
assert response.status == 200
+ async def test_init_script_is_available_in_page_action(self, urls, tmp_path):
+ """Test init_script globals are visible to the automation hook."""
+ script_path = tmp_path / "init_script.js"
+ script_path.write_text("window.calculateSum = function(a, b) { return a + b; };", encoding="utf-8")
+ results = []
+
+ async def check_execution_context(page):
+ results.append(await page.evaluate("() => window.calculateSum(1, 1)"))
+
+ async with AsyncStealthySession(init_script=str(script_path), page_action=check_execution_context) as session:
+ response = await session.fetch(urls["html"])
+
+ assert response.status == 200
+ assert results == [2]
+
async def test_error_handling_in_fetch(self, urls):
"""Test error handling during fetch"""
async with AsyncStealthySession() as session: