🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(cf solver): avoid missing-widget waits and bound Cloudflare solver loops
Karim shoairc0634a0e9001
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex 875b5cb..225f032 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -129,7 +129,9 @@ class StealthySession(SyncSession, StealthySessionMixin):else:log.info(f'The turnstile version discovered is "{challenge_type}"')if challenge_type == "non-interactive":- while self._detect_cloudflare(ResponseFactory._get_page_content(page)) == "non-interactive":+ for _ in range(10):+ if self._detect_cloudflare(ResponseFactory._get_page_content(page)) != "non-interactive":+ breaklog.info("Waiting for Cloudflare wait page to disappear.")page.wait_for_timeout(1000)page.wait_for_load_state()@@ -156,9 +158,12 @@ class StealthySession(SyncSession, StealthySessionMixin):self._wait_for_page_stability(iframe, True, False)if challenge_type != "embedded":- while not iframe.frame_element().is_visible():- # Double-checking that the iframe is loaded+ for _ in range(20):+ if iframe.frame_element().is_visible():+ breakpage.wait_for_timeout(500)+ else:+ return self._cloudflare_solver(page, _attempts + 1)outer_box = iframe.frame_element().bounding_box()@@ -167,7 +172,9 @@ class StealthySession(SyncSession, StealthySessionMixin):log.info("Cloudflare captcha is solved")return None- outer_box = page.locator(box_selector).last.bounding_box()+ if elements := page.locator(box_selector).last.element_handles():+ outer_box = elements[0].bounding_box()+ elements[0].dispose()if not outer_box:page.wait_for_timeout(1000)return self._cloudflare_solver(page, _attempts + 1)@@ -416,7 +423,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):else:log.info(f'The turnstile version discovered is "{challenge_type}"')if challenge_type == "non-interactive":- while self._detect_cloudflare(await ResponseFactory._get_async_page_content(page)) == "non-interactive":+ for _ in range(10):+ if (+ self._detect_cloudflare(await ResponseFactory._get_async_page_content(page))+ != "non-interactive"+ ):+ breaklog.info("Waiting for Cloudflare wait page to disappear.")await page.wait_for_timeout(1000)await page.wait_for_load_state()@@ -443,9 +455,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):await self._wait_for_page_stability(iframe, True, False)if challenge_type != "embedded":- while not await (await iframe.frame_element()).is_visible():- # Double-checking that the iframe is loaded+ for _ in range(20):+ if await (await iframe.frame_element()).is_visible():+ breakawait page.wait_for_timeout(500)+ else:+ return await self._cloudflare_solver(page, _attempts + 1)outer_box = await (await iframe.frame_element()).bounding_box()@@ -454,7 +469,9 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):log.info("Cloudflare captcha is solved")return None- outer_box = await page.locator(box_selector).last.bounding_box()+ if elements := await page.locator(box_selector).last.element_handles():+ outer_box = await elements[0].bounding_box()+ await elements[0].dispose()if not outer_box:await page.wait_for_timeout(1000)return await self._cloudflare_solver(page, _attempts + 1)diff --git a/tests/fetchers/test_cloudflare_optional_widget.py b/tests/fetchers/test_cloudflare_optional_widget.pynew file mode 100644index 0000000..80d65ff--- /dev/null+++ b/tests/fetchers/test_cloudflare_optional_widget.py@@ -0,0 +1,221 @@+import asyncio+from unittest.mock import AsyncMock, MagicMock++import pytest+from patchright.sync_api import Error as PatchrightError, TimeoutError as PlaywrightTimeoutError++from scrapling.core._types import Any+from scrapling.engines._browsers import _stealth+++CHALLENGE = "<script>window._cf_chl_opt = {cType: 'interactive'};</script>"+CLEAN = "<p>Done</p>"+++def call(value, asynchronous):+ return asyncio.run(value) if asynchronous else value+++def setup(monkeypatch, asynchronous, matches):+ session_type = _stealth.AsyncStealthySession if asynchronous else _stealth.StealthySession+ mock = AsyncMock if asynchronous else MagicMock+ monkeypatch.setattr(session_type, "_wait_for_networkidle", mock())+ monkeypatch.setattr(session_type, "_wait_for_page_stability", mock())+ session = object.__new__(session_type)+ state: dict[str, Any] = {"content": CHALLENGE, "elapsed": 0.0}+ page, element = MagicMock(), MagicMock()++ def sleep(milliseconds):+ state["elapsed"] += milliseconds+ assert state["elapsed"] <= 60_000, "Solver did not stop polling"++ page.content = mock(side_effect=lambda: state["content"])+ page.wait_for_timeout = mock(side_effect=sleep)+ page.wait_for_load_state = mock()+ page.mouse.click = mock(side_effect=lambda *args, **kwargs: state.update(content=CLEAN))+ page.frame.return_value = None+ element.bounding_box = mock(return_value={"x": 100, "y": 200})+ element.dispose = mock()+ page.locator.return_value.last.element_handles = mock(side_effect=matches(element, state))+ return session, page, element, state++[email protected]("asynchronous", [False, True])+def test_challenge_clears_during_absent_widget_lookup(monkeypatch, asynchronous):+ def matches(element, state):+ def absent():+ state["content"] = CLEAN+ return []++ return absent++ session, page, element, state = setup(monkeypatch, asynchronous, matches)++ def auto_wait():+ state.update(content=CLEAN, elapsed=state["elapsed"] + 60_000)+ raise PlaywrightTimeoutError("Waiting for missing locator")++ mock = AsyncMock if asynchronous else MagicMock+ page.locator.return_value.last.bounding_box = mock(side_effect=auto_wait)+ assert call(session._cloudflare_solver(page), asynchronous) is None+ assert state["elapsed"] == 11_000+ page.locator.return_value.last.bounding_box.assert_not_called()+ page.mouse.click.assert_not_called()++[email protected]("asynchronous", [False, True])+def test_permanently_missing_widget_uses_existing_attempt_cap(monkeypatch, asynchronous):+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])+ assert call(session._cloudflare_solver(page), asynchronous) is None+ assert page.locator.return_value.last.element_handles.call_count == _stealth.__CF_MAX_SOLVE_ATTEMPTS__+ assert state["elapsed"] == 33_000+ page.mouse.click.assert_not_called()++[email protected]("asynchronous", [False, True])+def test_late_widget_is_clicked_on_next_attempt(monkeypatch, asynchronous):+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: [[], [element]])+ assert call(session._cloudflare_solver(page), asynchronous) is None+ assert page.locator.return_value.last.element_handles.call_count == 2+ assert page.mouse.click.call_count == 1+ element.dispose.assert_called_once()++[email protected]("asynchronous", [False, True])+def test_hidden_widget_retries_and_disposes_handles(monkeypatch, asynchronous):+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [element])+ element.bounding_box.return_value = None+ assert call(session._cloudflare_solver(page), asynchronous) is None+ assert element.dispose.call_count == _stealth.__CF_MAX_SOLVE_ATTEMPTS__+ page.mouse.click.assert_not_called()++[email protected]("asynchronous", [False, True])+def test_query_errors_are_not_hidden(monkeypatch, asynchronous):+ session, page, element, state = setup(+ monkeypatch, asynchronous, lambda element, state: PatchrightError("Page closed")+ )+ with pytest.raises(PatchrightError, match="Page closed"):+ call(session._cloudflare_solver(page), asynchronous)+ page.mouse.click.assert_not_called()+++PASSIVE = "<script>window._cf_chl_opt = {cType: 'non-interactive'};</script>"++[email protected]("asynchronous", [False, True])+def test_passive_challenge_stops_after_existing_attempt_limit(monkeypatch, asynchronous):+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])+ state["content"] = PASSIVE+ logger = MagicMock()+ monkeypatch.setattr(_stealth, "log", logger)++ assert call(session._cloudflare_solver(page), asynchronous) is None+ assert page.wait_for_timeout.call_count == 30+ assert page.wait_for_load_state.call_count == 30+ assert state["elapsed"] == 30_000+ page.mouse.click.assert_not_called()+ logger.error.assert_called_once()+ assert all(args.args != ("Cloudflare captcha is solved",) for args in logger.info.call_args_list)++[email protected]("asynchronous", [False, True])[email protected]("clear_after", [1000, 10_000, 30_000])+def test_passive_clearance_is_seen_after_last_sleep(monkeypatch, asynchronous, clear_after):+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])+ state["content"] = PASSIVE++ def sleep(milliseconds):+ state["elapsed"] += milliseconds+ if state["elapsed"] >= clear_after:+ state["content"] = CLEAN++ page.wait_for_timeout.side_effect = sleep+ assert call(session._cloudflare_solver(page), asynchronous) is None+ assert state["elapsed"] == clear_after+ page.mouse.click.assert_not_called()++[email protected]("asynchronous", [False, True])+def test_passive_challenge_switches_to_interactive(monkeypatch, asynchronous):+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [element])+ state["content"] = PASSIVE++ def sleep(milliseconds):+ state["elapsed"] += milliseconds+ if state["content"] == PASSIVE:+ state["content"] = CHALLENGE++ page.wait_for_timeout.side_effect = sleep+ assert call(session._cloudflare_solver(page), asynchronous) is None+ page.mouse.click.assert_called_once()+ assert state["content"] == CLEAN+++def setup_hidden_frame(page, asynchronous, visible):+ mock = AsyncMock if asynchronous else MagicMock+ frame, element = MagicMock(), MagicMock()+ frame.frame_element = mock(return_value=element)+ element.is_visible = mock(side_effect=visible)+ element.bounding_box = mock(return_value={"x": 100, "y": 200})+ page.frame.return_value = frame+ return frame, element++[email protected]("asynchronous", [False, True])+def test_hidden_frame_stops_without_clicking(monkeypatch, asynchronous):+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])+ frame, frame_element = setup_hidden_frame(page, asynchronous, lambda: False)+ logger = MagicMock()+ monkeypatch.setattr(_stealth, "log", logger)++ assert call(session._cloudflare_solver(page), asynchronous) is None+ assert page.wait_for_timeout.call_count == 60+ assert state["elapsed"] == 30_000+ page.mouse.click.assert_not_called()+ frame_element.bounding_box.assert_not_called()+ page.locator.assert_not_called()+ logger.error.assert_called_once()+ assert all(args.args != ("Cloudflare captcha is solved",) for args in logger.info.call_args_list)++[email protected]("asynchronous", [False, True])[email protected]("visible_after", [0, 500, 10_500, 29_500])+def test_late_frame_uses_existing_attempts(monkeypatch, asynchronous, visible_after):+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])+ frame, frame_element = setup_hidden_frame(page, asynchronous, lambda: state["elapsed"] >= visible_after)++ assert call(session._cloudflare_solver(page), asynchronous) is None+ assert state["elapsed"] == visible_after+ page.mouse.click.assert_called_once()+ assert state["content"] == CLEAN++[email protected]("asynchronous", [False, True])+def test_cleared_challenge_with_hidden_frame_is_not_clicked(monkeypatch, asynchronous):+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])+ frame, frame_element = setup_hidden_frame(page, asynchronous, lambda: False)++ def sleep(milliseconds):+ state["elapsed"] += milliseconds+ state["content"] = CLEAN++ page.wait_for_timeout.side_effect = sleep+ assert call(session._cloudflare_solver(page), asynchronous) is None+ assert state["elapsed"] == 10_000+ page.mouse.click.assert_not_called()+ frame_element.bounding_box.assert_not_called()++[email protected]("asynchronous", [False, True])+def test_passive_wait_preserves_load_state_check(monkeypatch, asynchronous):+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])+ state["content"] = PASSIVE++ def wait_for_load():+ state["content"] = CLEAN++ page.wait_for_load_state.side_effect = wait_for_load+ assert call(session._cloudflare_solver(page), asynchronous) is None+ page.wait_for_load_state.assert_called_once()+ page.mouse.click.assert_not_called()diff --git a/tests/fetchers/test_cloudflare_solver.py b/tests/fetchers/test_cloudflare_solver.pyindex 7184c4a..004c850 100644--- a/tests/fetchers/test_cloudflare_solver.py+++ b/tests/fetchers/test_cloudflare_solver.py@@ -29,7 +29,9 @@ class TestChallengeCleared:def _make_sync_page():page = MagicMock()page.frame.return_value = None- page.locator.return_value.last.bounding_box.return_value = {"x": 100, "y": 200}+ element = MagicMock()+ element.bounding_box.return_value = {"x": 100, "y": 200}+ page.locator.return_value.last.element_handles.return_value = [element]return page@@ -83,11 +85,15 @@ class TestAsyncSolver:page.frame.return_value = Nonepage.wait_for_timeout = AsyncMock()page.wait_for_load_state = AsyncMock()- page.locator.return_value.last.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})+ element = MagicMock()+ element.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})+ element.dispose = AsyncMock()+ page.locator.return_value.last.element_handles = AsyncMock(return_value=[element])page.mouse.click = AsyncMock(side_effect=lambda *args, **kwargs: state.update(content=CLEAN_PAGE))assert await session._cloudflare_solver(page) is Noneassert page.mouse.click.await_count == 1+ assert page.mouse.click.await_args is not Nonex, y = page.mouse.click.await_args.argsassert 126 <= x <= 128assert 225 <= y <= 227@@ -100,7 +106,10 @@ class TestAsyncSolver:page.frame.return_value = Nonepage.wait_for_timeout = AsyncMock()page.wait_for_load_state = AsyncMock()- page.locator.return_value.last.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})+ element = MagicMock()+ element.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})+ element.dispose = AsyncMock()+ page.locator.return_value.last.element_handles = AsyncMock(return_value=[element])page.mouse.click = AsyncMock()assert await session._cloudflare_solver(page) is None