🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: make stealth init scripts available to page actions
mehmet turac9a6d3a81baa8
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex c9d390c..14b63ca 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -146,6 +146,12 @@ class SyncSession:if network_idle:self._wait_for_networkidle(page)+ def _replay_init_script_for_page_action(self, page: Page) -> None:+ if self._config.init_script:+ # Make init_script globals visible to the automation execution context.+ with open(self._config.init_script, encoding="utf-8") as script_file:+ page.evaluate(script_file.read())+@staticmethoddef _create_response_handler(page_info: PageInfo[Page],@@ -333,6 +339,12 @@ class AsyncSession:if network_idle:await self._wait_for_networkidle(page)+ async def _replay_init_script_for_page_action(self, page: AsyncPage) -> None:+ if self._config.init_script:+ # Make init_script globals visible to the automation execution context.+ with open(self._config.init_script, encoding="utf-8") as script_file:+ await page.evaluate(script_file.read())+@staticmethoddef _create_response_handler(page_info: PageInfo[AsyncPage],diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex efa195e..39ba525 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -256,6 +256,11 @@ class StealthySession(SyncSession, StealthySessionMixin):self._wait_for_page_stability(page, params.load_dom, params.network_idle)if params.page_action:+ try:+ self._replay_init_script_for_page_action(page)+ except Exception as e: # pragma: no cover+ log.error(f"Error executing init_script before page_action: {e}")+try:_ = params.page_action(page)except Exception as e: # pragma: no cover@@ -532,6 +537,11 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):await self._wait_for_page_stability(page, params.load_dom, params.network_idle)if params.page_action:+ try:+ await self._replay_init_script_for_page_action(page)+ except Exception as e: # pragma: no cover+ log.error(f"Error executing init_script before page_action: {e}")+try:_ = await params.page_action(page)except Exception as e: # pragma: no coverdiff --git a/tests/fetchers/async/test_stealth_session.py b/tests/fetchers/async/test_stealth_session.pyindex db79002..b37d4a0 100644--- a/tests/fetchers/async/test_stealth_session.py+++ b/tests/fetchers/async/test_stealth_session.py@@ -78,6 +78,21 @@ class TestAsyncStealthySession:response = await session.fetch(urls["html"])assert response.status == 200+ async def test_init_script_is_available_in_page_action(self, urls, tmp_path):+ """Test init_script globals are visible to the automation hook."""+ script_path = tmp_path / "init_script.js"+ script_path.write_text("window.calculateSum = function(a, b) { return a + b; };", encoding="utf-8")+ results = []++ async def check_execution_context(page):+ results.append(await page.evaluate("() => window.calculateSum(1, 1)"))++ async with AsyncStealthySession(init_script=str(script_path), page_action=check_execution_context) as session:+ response = await session.fetch(urls["html"])++ assert response.status == 200+ assert results == [2]+async def test_error_handling_in_fetch(self, urls):"""Test error handling during fetch"""async with AsyncStealthySession() as session: