Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(cf solver): avoid missing-widget waits and bound Cloudflare solver loops

Karim shoairc0634a0e9001
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index 875b5cb..225f032 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -129,7 +129,9 @@ class StealthySession(SyncSession, StealthySessionMixin):
else:
log.info(f'The turnstile version discovered is "{challenge_type}"')
if challenge_type == "non-interactive":
- while self._detect_cloudflare(ResponseFactory._get_page_content(page)) == "non-interactive":
+ for _ in range(10):
+ if self._detect_cloudflare(ResponseFactory._get_page_content(page)) != "non-interactive":
+ break
log.info("Waiting for Cloudflare wait page to disappear.")
page.wait_for_timeout(1000)
page.wait_for_load_state()
@@ -156,9 +158,12 @@ class StealthySession(SyncSession, StealthySessionMixin):
self._wait_for_page_stability(iframe, True, False)
if challenge_type != "embedded":
- while not iframe.frame_element().is_visible():
- # Double-checking that the iframe is loaded
+ for _ in range(20):
+ if iframe.frame_element().is_visible():
+ break
page.wait_for_timeout(500)
+ else:
+ return self._cloudflare_solver(page, _attempts + 1)
outer_box = iframe.frame_element().bounding_box()
@@ -167,7 +172,9 @@ class StealthySession(SyncSession, StealthySessionMixin):
log.info("Cloudflare captcha is solved")
return None
- outer_box = page.locator(box_selector).last.bounding_box()
+ if elements := page.locator(box_selector).last.element_handles():
+ outer_box = elements[0].bounding_box()
+ elements[0].dispose()
if not outer_box:
page.wait_for_timeout(1000)
return self._cloudflare_solver(page, _attempts + 1)
@@ -416,7 +423,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
else:
log.info(f'The turnstile version discovered is "{challenge_type}"')
if challenge_type == "non-interactive":
- while self._detect_cloudflare(await ResponseFactory._get_async_page_content(page)) == "non-interactive":
+ for _ in range(10):
+ if (
+ self._detect_cloudflare(await ResponseFactory._get_async_page_content(page))
+ != "non-interactive"
+ ):
+ break
log.info("Waiting for Cloudflare wait page to disappear.")
await page.wait_for_timeout(1000)
await page.wait_for_load_state()
@@ -443,9 +455,12 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
await self._wait_for_page_stability(iframe, True, False)
if challenge_type != "embedded":
- while not await (await iframe.frame_element()).is_visible():
- # Double-checking that the iframe is loaded
+ for _ in range(20):
+ if await (await iframe.frame_element()).is_visible():
+ break
await page.wait_for_timeout(500)
+ else:
+ return await self._cloudflare_solver(page, _attempts + 1)
outer_box = await (await iframe.frame_element()).bounding_box()
@@ -454,7 +469,9 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
log.info("Cloudflare captcha is solved")
return None
- outer_box = await page.locator(box_selector).last.bounding_box()
+ if elements := await page.locator(box_selector).last.element_handles():
+ outer_box = await elements[0].bounding_box()
+ await elements[0].dispose()
if not outer_box:
await page.wait_for_timeout(1000)
return await self._cloudflare_solver(page, _attempts + 1)
diff --git a/tests/fetchers/test_cloudflare_optional_widget.py b/tests/fetchers/test_cloudflare_optional_widget.py
new file mode 100644
index 0000000..80d65ff
--- /dev/null
+++ b/tests/fetchers/test_cloudflare_optional_widget.py
@@ -0,0 +1,221 @@
+import asyncio
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from patchright.sync_api import Error as PatchrightError, TimeoutError as PlaywrightTimeoutError
+
+from scrapling.core._types import Any
+from scrapling.engines._browsers import _stealth
+
+
+CHALLENGE = "<script>window._cf_chl_opt = {cType: 'interactive'};</script>"
+CLEAN = "<p>Done</p>"
+
+
+def call(value, asynchronous):
+ return asyncio.run(value) if asynchronous else value
+
+
+def setup(monkeypatch, asynchronous, matches):
+ session_type = _stealth.AsyncStealthySession if asynchronous else _stealth.StealthySession
+ mock = AsyncMock if asynchronous else MagicMock
+ monkeypatch.setattr(session_type, "_wait_for_networkidle", mock())
+ monkeypatch.setattr(session_type, "_wait_for_page_stability", mock())
+ session = object.__new__(session_type)
+ state: dict[str, Any] = {"content": CHALLENGE, "elapsed": 0.0}
+ page, element = MagicMock(), MagicMock()
+
+ def sleep(milliseconds):
+ state["elapsed"] += milliseconds
+ assert state["elapsed"] <= 60_000, "Solver did not stop polling"
+
+ page.content = mock(side_effect=lambda: state["content"])
+ page.wait_for_timeout = mock(side_effect=sleep)
+ page.wait_for_load_state = mock()
+ page.mouse.click = mock(side_effect=lambda *args, **kwargs: state.update(content=CLEAN))
+ page.frame.return_value = None
+ element.bounding_box = mock(return_value={"x": 100, "y": 200})
+ element.dispose = mock()
+ page.locator.return_value.last.element_handles = mock(side_effect=matches(element, state))
+ return session, page, element, state
+
+
[email protected]("asynchronous", [False, True])
+def test_challenge_clears_during_absent_widget_lookup(monkeypatch, asynchronous):
+ def matches(element, state):
+ def absent():
+ state["content"] = CLEAN
+ return []
+
+ return absent
+
+ session, page, element, state = setup(monkeypatch, asynchronous, matches)
+
+ def auto_wait():
+ state.update(content=CLEAN, elapsed=state["elapsed"] + 60_000)
+ raise PlaywrightTimeoutError("Waiting for missing locator")
+
+ mock = AsyncMock if asynchronous else MagicMock
+ page.locator.return_value.last.bounding_box = mock(side_effect=auto_wait)
+ assert call(session._cloudflare_solver(page), asynchronous) is None
+ assert state["elapsed"] == 11_000
+ page.locator.return_value.last.bounding_box.assert_not_called()
+ page.mouse.click.assert_not_called()
+
+
[email protected]("asynchronous", [False, True])
+def test_permanently_missing_widget_uses_existing_attempt_cap(monkeypatch, asynchronous):
+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])
+ assert call(session._cloudflare_solver(page), asynchronous) is None
+ assert page.locator.return_value.last.element_handles.call_count == _stealth.__CF_MAX_SOLVE_ATTEMPTS__
+ assert state["elapsed"] == 33_000
+ page.mouse.click.assert_not_called()
+
+
[email protected]("asynchronous", [False, True])
+def test_late_widget_is_clicked_on_next_attempt(monkeypatch, asynchronous):
+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: [[], [element]])
+ assert call(session._cloudflare_solver(page), asynchronous) is None
+ assert page.locator.return_value.last.element_handles.call_count == 2
+ assert page.mouse.click.call_count == 1
+ element.dispose.assert_called_once()
+
+
[email protected]("asynchronous", [False, True])
+def test_hidden_widget_retries_and_disposes_handles(monkeypatch, asynchronous):
+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [element])
+ element.bounding_box.return_value = None
+ assert call(session._cloudflare_solver(page), asynchronous) is None
+ assert element.dispose.call_count == _stealth.__CF_MAX_SOLVE_ATTEMPTS__
+ page.mouse.click.assert_not_called()
+
+
[email protected]("asynchronous", [False, True])
+def test_query_errors_are_not_hidden(monkeypatch, asynchronous):
+ session, page, element, state = setup(
+ monkeypatch, asynchronous, lambda element, state: PatchrightError("Page closed")
+ )
+ with pytest.raises(PatchrightError, match="Page closed"):
+ call(session._cloudflare_solver(page), asynchronous)
+ page.mouse.click.assert_not_called()
+
+
+PASSIVE = "<script>window._cf_chl_opt = {cType: 'non-interactive'};</script>"
+
+
[email protected]("asynchronous", [False, True])
+def test_passive_challenge_stops_after_existing_attempt_limit(monkeypatch, asynchronous):
+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])
+ state["content"] = PASSIVE
+ logger = MagicMock()
+ monkeypatch.setattr(_stealth, "log", logger)
+
+ assert call(session._cloudflare_solver(page), asynchronous) is None
+ assert page.wait_for_timeout.call_count == 30
+ assert page.wait_for_load_state.call_count == 30
+ assert state["elapsed"] == 30_000
+ page.mouse.click.assert_not_called()
+ logger.error.assert_called_once()
+ assert all(args.args != ("Cloudflare captcha is solved",) for args in logger.info.call_args_list)
+
+
[email protected]("asynchronous", [False, True])
[email protected]("clear_after", [1000, 10_000, 30_000])
+def test_passive_clearance_is_seen_after_last_sleep(monkeypatch, asynchronous, clear_after):
+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])
+ state["content"] = PASSIVE
+
+ def sleep(milliseconds):
+ state["elapsed"] += milliseconds
+ if state["elapsed"] >= clear_after:
+ state["content"] = CLEAN
+
+ page.wait_for_timeout.side_effect = sleep
+ assert call(session._cloudflare_solver(page), asynchronous) is None
+ assert state["elapsed"] == clear_after
+ page.mouse.click.assert_not_called()
+
+
[email protected]("asynchronous", [False, True])
+def test_passive_challenge_switches_to_interactive(monkeypatch, asynchronous):
+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [element])
+ state["content"] = PASSIVE
+
+ def sleep(milliseconds):
+ state["elapsed"] += milliseconds
+ if state["content"] == PASSIVE:
+ state["content"] = CHALLENGE
+
+ page.wait_for_timeout.side_effect = sleep
+ assert call(session._cloudflare_solver(page), asynchronous) is None
+ page.mouse.click.assert_called_once()
+ assert state["content"] == CLEAN
+
+
+def setup_hidden_frame(page, asynchronous, visible):
+ mock = AsyncMock if asynchronous else MagicMock
+ frame, element = MagicMock(), MagicMock()
+ frame.frame_element = mock(return_value=element)
+ element.is_visible = mock(side_effect=visible)
+ element.bounding_box = mock(return_value={"x": 100, "y": 200})
+ page.frame.return_value = frame
+ return frame, element
+
+
[email protected]("asynchronous", [False, True])
+def test_hidden_frame_stops_without_clicking(monkeypatch, asynchronous):
+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])
+ frame, frame_element = setup_hidden_frame(page, asynchronous, lambda: False)
+ logger = MagicMock()
+ monkeypatch.setattr(_stealth, "log", logger)
+
+ assert call(session._cloudflare_solver(page), asynchronous) is None
+ assert page.wait_for_timeout.call_count == 60
+ assert state["elapsed"] == 30_000
+ page.mouse.click.assert_not_called()
+ frame_element.bounding_box.assert_not_called()
+ page.locator.assert_not_called()
+ logger.error.assert_called_once()
+ assert all(args.args != ("Cloudflare captcha is solved",) for args in logger.info.call_args_list)
+
+
[email protected]("asynchronous", [False, True])
[email protected]("visible_after", [0, 500, 10_500, 29_500])
+def test_late_frame_uses_existing_attempts(monkeypatch, asynchronous, visible_after):
+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])
+ frame, frame_element = setup_hidden_frame(page, asynchronous, lambda: state["elapsed"] >= visible_after)
+
+ assert call(session._cloudflare_solver(page), asynchronous) is None
+ assert state["elapsed"] == visible_after
+ page.mouse.click.assert_called_once()
+ assert state["content"] == CLEAN
+
+
[email protected]("asynchronous", [False, True])
+def test_cleared_challenge_with_hidden_frame_is_not_clicked(monkeypatch, asynchronous):
+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])
+ frame, frame_element = setup_hidden_frame(page, asynchronous, lambda: False)
+
+ def sleep(milliseconds):
+ state["elapsed"] += milliseconds
+ state["content"] = CLEAN
+
+ page.wait_for_timeout.side_effect = sleep
+ assert call(session._cloudflare_solver(page), asynchronous) is None
+ assert state["elapsed"] == 10_000
+ page.mouse.click.assert_not_called()
+ frame_element.bounding_box.assert_not_called()
+
+
[email protected]("asynchronous", [False, True])
+def test_passive_wait_preserves_load_state_check(monkeypatch, asynchronous):
+ session, page, element, state = setup(monkeypatch, asynchronous, lambda element, state: lambda: [])
+ state["content"] = PASSIVE
+
+ def wait_for_load():
+ state["content"] = CLEAN
+
+ page.wait_for_load_state.side_effect = wait_for_load
+ assert call(session._cloudflare_solver(page), asynchronous) is None
+ page.wait_for_load_state.assert_called_once()
+ page.mouse.click.assert_not_called()
diff --git a/tests/fetchers/test_cloudflare_solver.py b/tests/fetchers/test_cloudflare_solver.py
index 7184c4a..004c850 100644
--- a/tests/fetchers/test_cloudflare_solver.py
+++ b/tests/fetchers/test_cloudflare_solver.py
@@ -29,7 +29,9 @@ class TestChallengeCleared:
def _make_sync_page():
page = MagicMock()
page.frame.return_value = None
- page.locator.return_value.last.bounding_box.return_value = {"x": 100, "y": 200}
+ element = MagicMock()
+ element.bounding_box.return_value = {"x": 100, "y": 200}
+ page.locator.return_value.last.element_handles.return_value = [element]
return page
@@ -83,11 +85,15 @@ class TestAsyncSolver:
page.frame.return_value = None
page.wait_for_timeout = AsyncMock()
page.wait_for_load_state = AsyncMock()
- page.locator.return_value.last.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})
+ element = MagicMock()
+ element.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})
+ element.dispose = AsyncMock()
+ page.locator.return_value.last.element_handles = AsyncMock(return_value=[element])
page.mouse.click = AsyncMock(side_effect=lambda *args, **kwargs: state.update(content=CLEAN_PAGE))
assert await session._cloudflare_solver(page) is None
assert page.mouse.click.await_count == 1
+ assert page.mouse.click.await_args is not None
x, y = page.mouse.click.await_args.args
assert 126 <= x <= 128
assert 225 <= y <= 227
@@ -100,7 +106,10 @@ class TestAsyncSolver:
page.frame.return_value = None
page.wait_for_timeout = AsyncMock()
page.wait_for_load_state = AsyncMock()
- page.locator.return_value.last.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})
+ element = MagicMock()
+ element.bounding_box = AsyncMock(return_value={"x": 100, "y": 200})
+ element.dispose = AsyncMock()
+ page.locator.return_value.last.element_handles = AsyncMock(return_value=[element])
page.mouse.click = AsyncMock()
assert await session._cloudflare_solver(page) is None