Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

test: update tests accordingly

Karim shoair335beb932aeb
diff --git a/.bandit.yml b/.bandit.yml
index bd06507..d987c18 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -8,4 +8,5 @@ skips:
- B110 # Try, Except, Pass detected.
- B104 # Possible binding to all interfaces.
- B301 # Pickle and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue.
-- B108 # Probable insecure usage of temp file/directory.
\ No newline at end of file
+- B108 # Probable insecure usage of temp file/directory.
+- B102 # Use of exec detected.
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ec5342c..2684042 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -111,7 +111,7 @@ pip3 install git+https://github.com/D4Vinci/Scrapling.git@dev
## Tests
Scrapling includes a comprehensive test suite you can run with pytest. First, install all libraries and `pytest-plugins` listed in `tests/requirements.txt`. Then, running the tests will result in an output like this:
```bash
- $ pytest tests -n auto
+ $ pytest tests -m "not asyncio and not browser" -n auto
=============================== test session starts ===============================
platform darwin -- Python 3.13.8, pytest-8.4.2, pluggy-1.6.0 -- /Users/<redacted>/.venv/bin/python3.13
cachedir: .pytest_cache
@@ -128,13 +128,14 @@ Scrapling includes a comprehensive test suite you can run with pytest. First, in
```
Here, `-n auto` runs tests in parallel across multiple processes to increase speed.
-**Note:** You may need to run browser tests sequentially (`DynamicFetcher`/`StealthyFetcher`) to avoid conflicts. To run non-browser tests in parallel and browser tests separately:
+**Note:** Run browser and async tests sequentially to avoid conflicts. To run the other tests in parallel and these groups separately:
```bash
# Non-browser tests (parallel)
-pytest tests/ -k "not (DynamicFetcher or StealthyFetcher)" -n auto
+pytest tests/ -m "not asyncio and not browser" -n auto
# Browser tests (sequential)
-pytest tests/ -k "DynamicFetcher or StealthyFetcher"
+pytest tests/ -m "browser"
+pytest tests/ -m "asyncio and not browser"
```
Bonus: You can also see the test coverage with the `pytest` plugin below
diff --git a/pytest.ini b/pytest.ini
index 083e39a..b5914b6 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -4,4 +4,5 @@ asyncio_default_fixture_loop_scope = function
addopts = -p no:warnings --doctest-modules --ignore=setup.py --verbose
markers =
asyncio: marks tests as async
+ browser: tests that start a real browser
asyncio_fixture_scope = function
\ No newline at end of file
diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py
index c3161da..a0654e5 100644
--- a/tests/ai/test_ai_mcp.py
+++ b/tests/ai/test_ai_mcp.py
@@ -169,6 +169,7 @@ class TestOneShotTools:
assert len(sessions) == 4 and all(not session._is_alive for session in sessions)
assert not server._sessions
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_browser_fetch_once_renders_and_closes_through_mcp(self, monkeypatch):
sessions: list[AsyncStealthySession] = []
@@ -409,6 +410,7 @@ class TestSessionManagement:
def server(self):
return ScraplingMCPServer()
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_open_and_close_session(self, server):
"""Test opening and closing a stealthy session"""
@@ -423,6 +425,7 @@ class TestSessionManagement:
assert isinstance(closed, SessionClosedModel)
assert closed.session_id == session_id
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_list_sessions(self, server):
"""Test listing sessions"""
@@ -445,6 +448,7 @@ class TestSessionManagement:
# Cleanup
await server.close_session(session_id)
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_browser_fetch_reuses_the_session(self, server, test_url):
"""Test fetching a page twice through a persistent stealthy session"""
@@ -462,6 +466,7 @@ class TestSessionManagement:
await server.close_session(session_id)
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_browser_fetch_accepts_per_request_overrides(self, server, test_url):
"""A per-request override is honored on a session fetch"""
@@ -486,6 +491,7 @@ class TestSessionManagement:
with pytest.raises(ValueError, match="not found"):
await server.browser_fetch(url=test_url, session_id="nonexistent")
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_browser_fetch_with_closed_session(self, server, test_url):
"""Test fetching with a session that has been closed"""
@@ -496,6 +502,7 @@ class TestSessionManagement:
with pytest.raises(ValueError, match="not found"):
await server.browser_fetch(url=test_url, session_id=session_id)
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_browser_open_with_custom_id(self, server):
"""Test opening a session with a custom session_id"""
@@ -505,6 +512,7 @@ class TestSessionManagement:
await server.close_session("my-session")
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_browser_open_duplicate_id_raises(self, server):
"""Test that opening a session with a duplicate session_id raises an error"""
@@ -1058,6 +1066,7 @@ class TestScreenshot:
def server(self):
return ScraplingMCPServer()
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_screenshot_png_with_stealthy_session(self, server, test_url):
"""PNG screenshot via a stealthy session returns image and url content blocks"""
@@ -1074,6 +1083,7 @@ class TestScreenshot:
finally:
await server.close_session(opened.session_id)
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_screenshot_jpeg_with_quality(self, server, test_url):
"""JPEG screenshot with quality parameter via a stealthy session"""
@@ -1088,6 +1098,7 @@ class TestScreenshot:
finally:
await server.close_session(opened.session_id)
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_screenshot_full_page_taller_than_viewport(self, server):
"""full_page=True produces an image taller than the viewport-only capture"""
@@ -1112,6 +1123,7 @@ class TestScreenshot:
with pytest.raises(ValueError, match="not found"):
await server.browser_screenshot(session_id="does-not-exist")
+ @pytest.mark.browser
@pytest.mark.asyncio
async def test_screenshot_quality_with_png_raises(self, server, test_url):
"""quality is rejected when image_type is png"""
diff --git a/tests/ai/test_browser_actions.py b/tests/ai/test_browser_actions.py
index 265b8ae..ec41487 100644
--- a/tests/ai/test_browser_actions.py
+++ b/tests/ai/test_browser_actions.py
@@ -129,6 +129,7 @@ async def test_invalid_later_action_prevents_other_action_kinds(invalid: dict[st
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
@pytest.mark.parametrize("slowly", [False, True])
async def test_live_mixed_chain_resolves_targets_created_by_earlier_actions(slowly: bool) -> None:
@@ -188,6 +189,7 @@ async def test_live_mixed_chain_resolves_targets_created_by_earlier_actions(slow
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["textbox", "move", "click", "wait_element"])
async def test_live_stale_ref_stops_mixed_chain_without_retargeting(kind: str) -> None:
@@ -224,6 +226,7 @@ async def test_live_stale_ref_stops_mixed_chain_without_retargeting(kind: str) -
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
@pytest.mark.parametrize("use_ref", [False, True])
async def test_live_targets_keep_explicit_refs_and_ref_shaped_selectors_distinct(use_ref: bool) -> None:
diff --git a/tests/ai/test_browser_dialog_live.py b/tests/ai/test_browser_dialog_live.py
index fa95af4..8e3fc3d 100644
--- a/tests/ai/test_browser_dialog_live.py
+++ b/tests/ai/test_browser_dialog_live.py
@@ -8,6 +8,7 @@ from scrapling.core._types import Any
from tests.ai.test_browser_wait import _browser
@pytest.mark.asyncio
@pytest.mark.parametrize(
"expression, reply, expected",
@@ -50,6 +51,7 @@ async def test_dialog_live_native_results(expression: str, reply: dict[str, Any]
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
@pytest.mark.parametrize("trigger", ["selector", "coordinates", "keyboard"])
async def test_dialog_live_queued_replies_and_unexpected_dialog(trigger: str) -> None:
@@ -83,6 +85,7 @@ async def test_dialog_live_queued_replies_and_unexpected_dialog(trigger: str) ->
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
@pytest.mark.parametrize("slowly", [False, True])
async def test_dialog_live_new_reply_after_previous_trigger(slowly: bool) -> None:
@@ -112,6 +115,7 @@ async def test_dialog_live_new_reply_after_previous_trigger(slowly: bool) -> Non
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
async def test_dialog_live_keyboard_trigger() -> None:
async with _browser() as (client, session, page):
@@ -136,6 +140,7 @@ async def test_dialog_live_keyboard_trigger() -> None:
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
async def test_dialog_live_delayed_trigger_during_wait() -> None:
async with _browser() as (client, session, page):
@@ -162,6 +167,7 @@ async def test_dialog_live_delayed_trigger_during_wait() -> None:
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
@pytest.mark.parametrize("accept", [False, True])
async def test_dialog_live_beforeunload_navigation(accept: bool) -> None:
@@ -199,6 +205,7 @@ async def test_dialog_live_beforeunload_navigation(accept: bool) -> None:
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
@pytest.mark.parametrize("fail", [False, True])
async def test_dialog_live_unused_replies_do_not_reach_later_calls(fail: bool) -> None:
@@ -223,6 +230,7 @@ async def test_dialog_live_unused_replies_do_not_reach_later_calls(fail: bool) -
assert loads(await page.get_attribute("body", "data-result")) is False
@pytest.mark.asyncio
async def test_dialog_live_cancel_removes_unused_reply() -> None:
async with _browser() as (client, session, page):
diff --git a/tests/ai/test_browser_evaluate.py b/tests/ai/test_browser_evaluate.py
index d44d535..80f7338 100644
--- a/tests/ai/test_browser_evaluate.py
+++ b/tests/ai/test_browser_evaluate.py
@@ -249,6 +249,7 @@ async def test_browser_evaluate_reserves_page_until_cancelled(cancel_mode: str)
page.close.assert_not_called()
@pytest.mark.asyncio
async def test_browser_evaluate_live_expressions_promises_context_and_page_state() -> None:
server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))
diff --git a/tests/ai/test_browser_extract.py b/tests/ai/test_browser_extract.py
index 6965f51..3b41d12 100644
--- a/tests/ai/test_browser_extract.py
+++ b/tests/ai/test_browser_extract.py
@@ -171,6 +171,7 @@ async def test_browser_extract_releases_the_page_after_failure() -> None:
assert await server.browser_extract("browser") == SNAPSHOT
@pytest.mark.asyncio
@pytest.mark.parametrize("css_selector", [None, "main"])
async def test_browser_extract_live_mcp_round_trip(css_selector: str | None) -> None:
@@ -527,6 +528,7 @@ async def test_browser_extract_search_errors_are_value_errors(kwargs: dict[str,
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
async def test_browser_extract_search_live_refs_and_state() -> None:
server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))
diff --git a/tests/ai/test_browser_extract_content.py b/tests/ai/test_browser_extract_content.py
index 0ee78b3..4653f7a 100644
--- a/tests/ai/test_browser_extract_content.py
+++ b/tests/ai/test_browser_extract_content.py
@@ -244,6 +244,7 @@ async def test_browser_extract_content_cancellation_releases_page(target: str |
page.close.assert_not_called()
@pytest.mark.asyncio
async def test_browser_extract_content_live_formats_preserve_current_page() -> None:
async with _browser() as (client, session, page):
@@ -300,6 +301,7 @@ async def test_browser_extract_content_live_formats_preserve_current_page() -> N
assert before["scroll"] == 500 and not navigations and not requests
@pytest.mark.asyncio
async def test_browser_extract_content_live_selectors_refs_and_snapshot_scope() -> None:
async with _browser() as (client, session, page):
@@ -327,6 +329,7 @@ async def test_browser_extract_content_live_selectors_refs_and_snapshot_scope()
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
async def test_browser_extract_content_live_target_errors_allow_next_call() -> None:
async with _browser() as (client, session, page):
diff --git a/tests/ai/test_browser_fill_fields.py b/tests/ai/test_browser_fill_fields.py
index 14e1f50..a195182 100644
--- a/tests/ai/test_browser_fill_fields.py
+++ b/tests/ai/test_browser_fill_fields.py
@@ -510,6 +510,7 @@ async def _browser() -> AsyncGenerator[tuple[Client, Any, Any], None]:
assert not closed.is_error
@pytest.mark.asyncio
async def test_browser_actions_fields_live_mixed_fields_and_clearing() -> None:
async with _browser() as (client, session, page):
@@ -580,6 +581,7 @@ async def test_browser_actions_fields_live_mixed_fields_and_clearing() -> None:
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
async def test_browser_actions_fields_live_slow_typing_and_field_delays() -> None:
async with _browser() as (client, session, page):
@@ -634,6 +636,7 @@ async def test_browser_actions_fields_live_slow_typing_and_field_delays() -> Non
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
@pytest.mark.parametrize("slowly", [False, True])
async def test_browser_actions_fields_live_caret_selection_and_submit(slowly: bool) -> None:
@@ -681,6 +684,7 @@ async def test_browser_actions_fields_live_caret_selection_and_submit(slowly: bo
] == ["X", "Y", "z", "!"]
@pytest.mark.asyncio
async def test_browser_actions_fields_live_failed_field_preserves_prior_changes() -> None:
async with _browser() as (client, session, page):
diff --git a/tests/ai/test_browser_mouse.py b/tests/ai/test_browser_mouse.py
index 25e4176..b554f16 100644
--- a/tests/ai/test_browser_mouse.py
+++ b/tests/ai/test_browser_mouse.py
@@ -633,6 +633,7 @@ async def test_browser_mouse_chain_reports_failed_index_and_keeps_completed_acti
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
async def test_browser_mouse_live_native_events_and_navigation() -> None:
server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))
@@ -708,6 +709,7 @@ async def test_browser_mouse_live_native_events_and_navigation() -> None:
assert not closed.is_error
@pytest.mark.asyncio
@pytest.mark.parametrize("cancel_mode", ["task", "scope"])
@pytest.mark.parametrize("target_type", ["xy", "ref"])
@@ -774,6 +776,7 @@ async def test_browser_mouse_live_cancelled_click_releases_native_button(cancel_
assert not closed.is_error
@pytest.mark.asyncio
@pytest.mark.parametrize("tool", ["move", "click"])
async def test_browser_mouse_live_selectors_refs_and_scroll(tool: str) -> None:
@@ -849,6 +852,7 @@ async def test_browser_mouse_live_selectors_refs_and_scroll(tool: str) -> None:
assert not closed.is_error
@pytest.mark.asyncio
async def test_browser_mouse_click_live_timeout_releases_native_button() -> None:
server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))
diff --git a/tests/ai/test_browser_network.py b/tests/ai/test_browser_network.py
index 89646fa..cb2d98d 100644
--- a/tests/ai/test_browser_network.py
+++ b/tests/ai/test_browser_network.py
@@ -873,6 +873,7 @@ async def test_network_missing_id_and_search_error_reach_client() -> None:
assert result.is_error and "regular expression" in result.content[0].text
@pytest.mark.asyncio
async def test_network_live_capture_actions_navigation_and_reads() -> None:
server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))
diff --git a/tests/ai/test_browser_press_key.py b/tests/ai/test_browser_press_key.py
index 3863d86..0b3064c 100644
--- a/tests/ai/test_browser_press_key.py
+++ b/tests/ai/test_browser_press_key.py
@@ -208,6 +208,7 @@ async def _browser() -> AsyncGenerator[tuple[Client, Any, Any], None]:
assert not closed.is_error
@pytest.mark.asyncio
async def test_browser_actions_keys_live_focus_editing_and_shortcuts() -> None:
async with _browser() as (client, session, page):
diff --git a/tests/ai/test_browser_screenshot.py b/tests/ai/test_browser_screenshot.py
index 8abd249..276e5d5 100644
--- a/tests/ai/test_browser_screenshot.py
+++ b/tests/ai/test_browser_screenshot.py
@@ -427,6 +427,7 @@ async def test_browser_screenshot_reserves_page_until_cancelled(
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
async def test_browser_screenshot_live_preserves_actions_and_current_page() -> None:
server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))
diff --git a/tests/ai/test_browser_state.py b/tests/ai/test_browser_state.py
index 7d85d1e..526c18a 100644
--- a/tests/ai/test_browser_state.py
+++ b/tests/ai/test_browser_state.py
@@ -83,6 +83,7 @@ async def test_browser_state_requires_live_browser(operation: str, kind: str, me
session.load_state.assert_not_awaited()
@pytest.mark.asyncio
async def test_browser_state_live_round_trip(tmp_path: Path) -> None:
path, native_path = tmp_path / "state.json", tmp_path / "native.json"
@@ -118,6 +119,7 @@ async def test_browser_state_live_round_trip(tmp_path: Path) -> None:
assert await page.evaluate(READ_STATE) == SAVED_STATE
@pytest.mark.asyncio
async def test_browser_state_live_file_errors(tmp_path: Path) -> None:
invalid = tmp_path / "invalid.json"
diff --git a/tests/ai/test_browser_wait.py b/tests/ai/test_browser_wait.py
index 7e5b664..13f6bf4 100644
--- a/tests/ai/test_browser_wait.py
+++ b/tests/ai/test_browser_wait.py
@@ -339,6 +339,7 @@ async def _browser() -> AsyncGenerator[tuple[Client, Any, Any], None]:
assert not (await client.call_tool("close_session", {"session_id": "browser"})).is_error
@pytest.mark.asyncio
async def test_browser_wait_live_pause_and_element_states() -> None:
async with _browser() as (client, session, page):
@@ -396,6 +397,7 @@ async def test_browser_wait_live_pause_and_element_states() -> None:
assert session.page_pool.pages[0].state == "ready"
@pytest.mark.asyncio
async def test_browser_wait_live_current_document_load_states() -> None:
async with _browser() as (client, session, page):
@@ -452,6 +454,7 @@ async def test_browser_wait_live_current_document_load_states() -> None:
await asyncio.gather(task, return_exceptions=True)
@pytest.mark.asyncio
async def test_browser_wait_live_load_visible_hidden_chain() -> None:
async with _browser() as (client, session, page):
diff --git a/tests/ai/test_browser_wheel.py b/tests/ai/test_browser_wheel.py
index 2bd01d5..06654cc 100644
--- a/tests/ai/test_browser_wheel.py
+++ b/tests/ai/test_browser_wheel.py
@@ -215,6 +215,7 @@ async def _browser() -> AsyncGenerator[tuple[Client, Any, Any], None]:
assert not closed.is_error
@pytest.mark.asyncio
async def test_browser_wheel_live_scrolls_under_current_pointer() -> None:
async with _browser() as (client, session, page):
diff --git a/tests/cli/test_browser_commands.py b/tests/cli/test_browser_commands.py
new file mode 100644
index 0000000..fbcc39e
--- /dev/null
+++ b/tests/cli/test_browser_commands.py
@@ -0,0 +1,169 @@
+from inspect import Parameter, signature
+from pathlib import Path
+from unittest.mock import MagicMock, call, patch
+
+import pytest
+from click.testing import CliRunner
+
+from scrapling import cli
+from scrapling.core.shell import CustomShell
+from scrapling.core._shell_signatures import Signatures_map
+from scrapling.engines._browsers._types import StealthSession
+from scrapling.fetchers import StealthyFetcher
+from scrapling.parser import Selector
+
+
+ ("flags", "options"),
+ [
+ ([], {"block_webrtc": False, "solve_cloudflare": False, "allow_webgl": True, "hide_canvas": False}),
+ (
+ ["--block-webrtc", "--solve-cloudflare", "--block-webgl", "--hide-canvas"],
+ {"block_webrtc": True, "solve_cloudflare": True, "allow_webgl": False, "hide_canvas": True},
+ ),
+ (
+ ["--allow-webrtc", "--no-solve-cloudflare", "--allow-webgl", "--show-canvas"],
+ {"block_webrtc": False, "solve_cloudflare": False, "allow_webgl": True, "hide_canvas": False},
+ ),
+ ],
+)
+def test_fetch_stealth_options(tmp_path: Path, flags: list[str], options: dict[str, bool]) -> None:
+ with patch.object(StealthyFetcher, "fetch", return_value=Selector("<p>Content</p>")) as fetch:
+ result = CliRunner().invoke(
+ cli.main, ["extract", "fetch", "https://example.com", str(tmp_path / "page.txt"), *flags]
+ )
+ assert result.exit_code == 0, result.output
+ assert options.items() <= fetch.call_args.kwargs.items()
+
+
+def test_fetch_common_options(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("SCRAPLING_EXECUTABLE_PATH", raising=False)
+ output = tmp_path / "page.txt"
+ with patch.object(StealthyFetcher, "fetch", return_value=Selector("<p>Content</p><b>Other</b>")) as fetch:
+ result = CliRunner().invoke(
+ cli.main,
+ [
+ "extract",
+ "fetch",
+ "https://example.com",
+ str(output),
+ "--no-headless",
+ "--disable-resources",
+ "--network-idle",
+ "--pierce-shadow",
+ "--timeout",
+ "60000",
+ "--wait",
+ "50",
+ "--css-selector",
+ "p",
+ "--wait-selector",
+ "p",
+ "--locale",
+ "en-GB",
+ "--real-chrome",
+ "--proxy",
+ "http://localhost:8080",
+ "--extra-headers",
+ "X-Test: value",
+ "--executable-path",
+ "/opt/chromium",
+ "--dns-over-https",
+ "--block-ads",
+ ],
+ )
+ assert result.exit_code == 0, result.output
+ fetch.assert_called_once_with(
+ "https://example.com",
+ headless=False,
+ disable_resources=True,
+ network_idle=True,
+ pierce_shadow=True,
+ timeout=60000,
+ wait=50,
+ wait_selector="p",
+ locale="en-GB",
+ real_chrome=True,
+ proxy="http://localhost:8080",
+ extra_headers={"X-Test": "value"},
+ executable_path="/opt/chromium",
+ dns_over_https=True,
+ block_ads=True,
+ block_webrtc=False,
+ solve_cloudflare=False,
+ allow_webgl=True,
+ hide_canvas=False,
+ )
+ assert output.read_text() == "Content"
+
+
+def test_browser_command_help_and_removed_command() -> None:
+ runner = CliRunner()
+ result = runner.invoke(cli.main, ["extract", "fetch", "--help"])
+ assert result.exit_code == 0
+ assert "StealthyFetcher" in result.output
+ for option in ("--block-webrtc", "--solve-cloudflare", "--allow-webgl", "--hide-canvas"):
+ assert option in result.output
+ result = runner.invoke(cli.main, ["extract", "stealthy-fetch", "--help"])
+ assert result.exit_code == 2
+ assert "No such command 'stealthy-fetch'" in result.output
+ assert not hasattr(cli, "stealthy_fetch")
+
+
[email protected](("installed", "force"), [(False, False), (True, False), (True, True), (False, True)])
+def test_install_playwright_keeps_marker(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, installed: bool, force: bool
+) -> None:
+ marker = tmp_path / ".scrapling_dependencies_installed"
+ if installed:
+ marker.touch()
+ monkeypatch.setattr(cli, "__PACKAGE_DIR__", tmp_path)
+ with patch("scrapling.cli.check_output") as execute, patch("tld.utils.update_tld_names") as update:
+ result = CliRunner().invoke(cli.main, ["install", *(["--force"] if force else [])])
+ assert result.exit_code == 0, result.output
+ assert marker.exists()
+ if force or not installed:
+ assert execute.call_args_list == [
+ call([cli.python_executable, "-m", "playwright", "install", "chromium"], shell=False),
+ call([cli.python_executable, "-m", "playwright", "install-deps", "chromium"], shell=False),
+ ]
+ update.assert_called_once_with(fail_silently=True)
+ assert "Installing browsers" in result.output
+ else:
+ execute.assert_not_called()
+ update.assert_not_called()
+ assert result.output.strip() == "The dependencies are already installed"
+
+
+def test_shell_fetch_namespace_and_completion() -> None:
+ shell = CustomShell(code="")
+ namespace = shell.get_namespace()
+ assert namespace["StealthyFetcher"] is StealthyFetcher
+ assert {"StealthySession", "AsyncStealthySession", "fetch"} <= namespace.keys()
+ assert (
+ not {"DynamicFetcher", "DynamicSession", "AsyncDynamicSession", "PlayWrightFetcher", "stealthy_fetch"}
+ & namespace.keys()
+ )
+ parameters = signature(namespace["fetch"]).parameters
+ assert set(parameters) == {"url", *StealthSession.__annotations__}
+ assert all(parameters[name].kind is Parameter.KEYWORD_ONLY for name in StealthSession.__annotations__)
+ assert Signatures_map["fetch"] == StealthSession.__annotations__
+ assert "stealthy_fetch" not in Signatures_map
+ assert "Shortcut for `StealthyFetcher.fetch`" in shell.banner()
+ assert "Dynamic" not in shell.banner()
+ assert "stealthy_fetch" not in shell.banner()
+
+
+def test_shell_fetch_updates_page_history() -> None:
+ pages = [Selector(f"<p>{index}</p>") for index in range(7)]
+ shell = CustomShell(code="")
+ shell.shell = MagicMock(user_ns={})
+ with patch.object(StealthyFetcher, "fetch", side_effect=pages) as fetch:
+ namespace = shell.get_namespace()
+ for page in pages:
+ assert namespace["fetch"]("https://example.com", solve_cloudflare=True) is page
+ assert fetch.call_count == len(pages)
+ fetch.assert_called_with("https://example.com", solve_cloudflare=True)
+ assert shell.page is pages[-1]
+ assert shell.pages == pages[-5:]
+ assert shell.shell.user_ns == {"page": pages[-1], "response": pages[-1], "pages": pages[-5:]}
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index dde4db6..27d164c 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -5,7 +5,7 @@ import pytest_httpbin
from scrapling.parser import Selector
from scrapling import __version__
-from scrapling.cli import main, shell, mcp, get, post, put, delete, fetch, stealthy_fetch
+from scrapling.cli import main, shell, mcp, get, post, put, delete, fetch
@pytest_httpbin.use_class_based_httpbin
@@ -215,33 +215,19 @@ class TestCLI:
"""Test extract fetch command"""
output_file = tmp_path / "output.txt"
- with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:
- mock_response = configure_selector_mock()
- mock_fetch.return_value = mock_response
-
- result = runner.invoke(fetch, [html_url, str(output_file), "--headless", "--timeout", "60000"])
- assert result.exit_code == 0
-
- def test_extract_stealthy_fetch_command(self, runner, tmp_path, html_url):
- """Test extract fetch command"""
- output_file = tmp_path / "output.md"
-
with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
mock_response = configure_selector_mock()
mock_fetch.return_value = mock_response
- result = runner.invoke(
- stealthy_fetch,
- [html_url, str(output_file), "--headless", "--css-selector", "body", "--timeout", "60000"],
- )
+ result = runner.invoke(fetch, [html_url, str(output_file), "--headless", "--timeout", "60000"])
assert result.exit_code == 0
def test_extract_fetch_with_executable_path(self, runner, tmp_path, html_url, monkeypatch):
- """Test that --executable-path is passed through to DynamicFetcher and wins over the environment variable"""
+ """Test that --executable-path is passed through to StealthyFetcher and wins over the environment variable"""
output_file = tmp_path / "output.html"
monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/env-chromium")
- with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:
+ with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
mock_fetch.return_value = configure_selector_mock()
result = runner.invoke(fetch, [html_url, str(output_file), "--executable-path", "/opt/custom-chromium"])
@@ -253,7 +239,7 @@ class TestCLI:
output_file = tmp_path / "output.html"
monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/env-chromium")
- with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:
+ with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
mock_fetch.return_value = configure_selector_mock()
result = runner.invoke(fetch, [html_url, str(output_file)])
@@ -265,35 +251,13 @@ class TestCLI:
output_file = tmp_path / "output.html"
monkeypatch.delenv("SCRAPLING_EXECUTABLE_PATH", raising=False)
- with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:
+ with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
mock_fetch.return_value = configure_selector_mock()
result = runner.invoke(fetch, [html_url, str(output_file)])
assert result.exit_code == 0
assert "executable_path" not in mock_fetch.call_args.kwargs
- def test_extract_stealthy_fetch_with_executable_path(self, runner, tmp_path, html_url, monkeypatch):
- """Test that --executable-path and the environment fallback work for stealthy_fetch too"""
- output_file = tmp_path / "output.html"
- monkeypatch.delenv("SCRAPLING_EXECUTABLE_PATH", raising=False)
-
- with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
- mock_fetch.return_value = configure_selector_mock()
-
- result = runner.invoke(
- stealthy_fetch, [html_url, str(output_file), "--executable-path", "/opt/custom-chromium"]
- )
- assert result.exit_code == 0
- assert mock_fetch.call_args.kwargs["executable_path"] == "/opt/custom-chromium"
-
- monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/env-chromium")
- with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
- mock_fetch.return_value = configure_selector_mock()
-
- result = runner.invoke(stealthy_fetch, [html_url, str(output_file)])
- assert result.exit_code == 0
- assert mock_fetch.call_args.kwargs["executable_path"] == "/opt/env-chromium"
-
def test_invalid_arguments(self, runner, html_url):
"""Test invalid arguments handling"""
# Missing required arguments
diff --git a/tests/cli/test_shadow_cli.py b/tests/cli/test_shadow_cli.py
index 097a5e2..d7e8121 100644
--- a/tests/cli/test_shadow_cli.py
+++ b/tests/cli/test_shadow_cli.py
@@ -8,24 +8,20 @@ from scrapling.cli import main
from scrapling.parser import Selector
[email protected](("command", "fetcher"), [("fetch", "DynamicFetcher"), ("stealthy-fetch", "StealthyFetcher")])
@pytest.mark.parametrize(
("flags", "enabled"), [([], False), (["--pierce-shadow"], True), (["--no-pierce-shadow"], False)]
)
-def test_shadow_option_reaches_fetcher(
- tmp_path: Path, command: str, fetcher: str, flags: list[str], enabled: bool
-) -> None:
+def test_shadow_option_reaches_fetcher(tmp_path: Path, flags: list[str], enabled: bool) -> None:
output = tmp_path / "page.txt"
- with patch(f"scrapling.fetchers.{fetcher}.fetch", return_value=Selector("<p>Content</p>")) as fetch:
- result = CliRunner().invoke(main, ["extract", command, "https://example.com", str(output), *flags])
+ with patch("scrapling.fetchers.StealthyFetcher.fetch", return_value=Selector("<p>Content</p>")) as fetch:
+ result = CliRunner().invoke(main, ["extract", "fetch", "https://example.com", str(output), *flags])
assert result.exit_code == 0, result.output
assert fetch.call_args.kwargs["pierce_shadow"] is enabled
assert output.read_text() == "Content"
[email protected]("command", ["fetch", "stealthy-fetch"])
-def test_shadow_option_in_browser_help(command: str) -> None:
- result = CliRunner().invoke(main, ["extract", command, "--help"])
+def test_shadow_option_in_browser_help() -> None:
+ result = CliRunner().invoke(main, ["extract", "fetch", "--help"])
assert result.exit_code == 0
assert "--pierce-shadow / --no-pierce-shadow" in result.output
diff --git a/tests/cli/test_shell_functionality.py b/tests/cli/test_shell_functionality.py
index d23fa22..995e657 100644
--- a/tests/cli/test_shell_functionality.py
+++ b/tests/cli/test_shell_functionality.py
@@ -2,6 +2,7 @@ import pytest
from unittest.mock import patch, MagicMock
from scrapling.parser import Selector
+from scrapling.core._types import cast, extraction_types
from scrapling.core.shell import CustomShell, CurlParser, Convertor
@@ -15,11 +16,11 @@ class TestCurlParser:
def test_basic_curl_parse(self, parser):
"""Test parsing basic curl commands"""
# Simple GET
- curl_cmd = 'curl https://example.com'
+ curl_cmd = "curl https://example.com"
request = parser.parse(curl_cmd)
- assert request.url == 'https://example.com'
- assert request.method == 'get'
+ assert request.url == "https://example.com"
+ assert request.method == "get"
assert request.data is None
def test_curl_with_headers(self, parser):
@@ -30,8 +31,8 @@ class TestCurlParser:
request = parser.parse(curl_cmd)
- assert request.headers['User-Agent'] == 'Mozilla/5.0'
- assert request.headers['Accept'] == 'application/json'
+ assert request.headers["User-Agent"] == "Mozilla/5.0"
+ assert request.headers["Accept"] == "application/json"
def test_curl_with_data(self, parser):
"""Test parsing curl with data"""
@@ -39,8 +40,8 @@ class TestCurlParser:
curl_cmd = 'curl https://example.com -X POST -d "key=value&foo=bar"'
request = parser.parse(curl_cmd)
- assert request.method == 'post'
- assert request.data == 'key=value&foo=bar'
+ assert request.method == "post"
+ assert request.data == "key=value&foo=bar"
# JSON data
curl_cmd = """curl https://example.com -X POST --data-raw '{"key": "value"}'"""
@@ -56,24 +57,24 @@ class TestCurlParser:
request = parser.parse(curl_cmd)
- assert request.cookies['session'] == 'abc123'
- assert request.cookies['user'] == 'john'
- assert request.cookies['extra'] == 'cookie'
+ assert request.cookies["session"] == "abc123"
+ assert request.cookies["user"] == "john"
+ assert request.cookies["extra"] == "cookie"
def test_curl_with_proxy(self, parser):
"""Test parsing curl with proxy"""
- curl_cmd = 'curl https://example.com -x http://proxy:8080 -U user:pass'
+ curl_cmd = "curl https://example.com -x http://proxy:8080 -U user:pass"
request = parser.parse(curl_cmd)
- assert 'http://user:pass@proxy:8080' in request.proxy['http']
+ assert "http://user:pass@proxy:8080" in request.proxy["http"]
def test_curl2fetcher(self, parser):
"""Test converting curl to fetcher request"""
- with patch('scrapling.fetchers.Fetcher.get') as mock_get:
+ with patch("scrapling.fetchers.Fetcher.get") as mock_get:
mock_response = MagicMock()
mock_get.return_value = mock_response
- curl_cmd = 'curl https://example.com'
+ curl_cmd = "curl https://example.com"
_ = parser.convert2fetcher(curl_cmd)
mock_get.assert_called_once()
@@ -82,7 +83,7 @@ class TestCurlParser:
"""Test handling invalid curl commands"""
# Invalid format
with pytest.raises(AttributeError):
- parser.parse('not a curl command')
+ parser.parse("not a curl command")
class TestConvertor:
@@ -129,11 +130,7 @@ class TestConvertor:
def test_extract_with_selector(self, sample_html):
"""Test extracting with CSS selector"""
page = Selector(sample_html)
- content = list(Convertor._extract_content(
- page,
- "text",
- css_selector=".content"
- ))
+ content = list(Convertor._extract_content(page, "text", css_selector=".content"))
assert len(content) > 0
@@ -162,7 +159,7 @@ class TestConvertor:
# Invalid extraction type
with pytest.raises(ValueError):
- list(Convertor._extract_content(page, "invalid"))
+ list(Convertor._extract_content(page, cast(extraction_types, "invalid")))
# Invalid filename
with pytest.raises(ValueError):
@@ -190,9 +187,9 @@ class TestCustomShell:
namespace = shell.get_namespace()
# Check all expected functions/classes are available
- assert 'get' in namespace
- assert 'post' in namespace
- assert 'Fetcher' in namespace
- assert 'DynamicFetcher' in namespace
- assert 'view' in namespace
- assert 'uncurl' in namespace
+ assert "get" in namespace
+ assert "post" in namespace
+ assert "Fetcher" in namespace
+ assert "StealthyFetcher" in namespace
+ assert "view" in namespace
+ assert "uncurl" in namespace
diff --git a/tests/fetchers/async/test_dynamic.py b/tests/fetchers/async/test_dynamic.py
deleted file mode 100644
index 3fe38cc..0000000
--- a/tests/fetchers/async/test_dynamic.py
+++ /dev/null
@@ -1,90 +0,0 @@
-import pytest
-import pytest_httpbin
-
-from scrapling import DynamicFetcher
-
-DynamicFetcher.adaptive = True
-
-
-@pytest_httpbin.use_class_based_httpbin
-class TestDynamicFetcherAsync:
- @pytest.fixture
- def fetcher(self):
- return DynamicFetcher
-
- @pytest.fixture
- def urls(self, httpbin):
- return {
- "status_200": f"{httpbin.url}/status/200",
- "status_404": f"{httpbin.url}/status/404",
- "status_501": f"{httpbin.url}/status/501",
- "basic_url": f"{httpbin.url}/get",
- "html_url": f"{httpbin.url}/html",
- "delayed_url": f"{httpbin.url}/delay/10",
- "cookies_url": f"{httpbin.url}/cookies/set/test/value",
- }
-
- @pytest.mark.asyncio
- async def test_basic_fetch(self, fetcher, urls):
- """Test doing a basic fetch request with multiple statuses"""
- response = await fetcher.async_fetch(urls["status_200"])
- assert response.status == 200
-
- @pytest.mark.asyncio
- async def test_cookies_loading(self, fetcher, urls):
- """Test if cookies are set after the request"""
- response = await fetcher.async_fetch(urls["cookies_url"])
- cookies = {response.cookies[0]['name']: response.cookies[0]['value']}
- assert cookies == {"test": "value"}
-
- @pytest.mark.asyncio
- async def test_automation(self, fetcher, urls):
- """Test if automation breaks the code or not"""
-
- async def scroll_page(page):
- await page.mouse.wheel(10, 0)
- await page.mouse.move(100, 400)
- await page.mouse.up()
- return page
-
- response = await fetcher.async_fetch(urls["html_url"], page_action=scroll_page)
- assert response.status == 200
-
- @pytest.mark.parametrize(
- "kwargs",
- [
- {"real_chrome": True, "disable_resources": True},
- {"wait_selector": "h1", "wait_selector_state": "attached"},
- {"wait_selector": "h1", "wait_selector_state": "visible"},
- {
- "google_search": True,
- "real_chrome": True,
- "wait": 10,
- "locale": "en-US",
- "extra_headers": {"ayo": ""},
- "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0",
- "cookies": [{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],
- "network_idle": True,
- "selector_config": {"keep_comments": False, "keep_cdata": False},
- },
- ],
- )
- @pytest.mark.asyncio
- async def test_properties(self, fetcher, urls, kwargs):
- """Test if different arguments break the code or not"""
- response = await fetcher.async_fetch(urls["html_url"], **kwargs)
- assert response.status == 200
-
- @pytest.mark.asyncio
- async def test_cdp_url_invalid(self, fetcher, urls):
- """Test if invalid CDP URLs raise appropriate exceptions"""
- with pytest.raises(TypeError):
- await fetcher.async_fetch(urls["html_url"], cdp_url="blahblah")
-
- with pytest.raises(TypeError):
- await fetcher.async_fetch(
- urls["html_url"], cdp_url="blahblah"
- )
-
- with pytest.raises(Exception):
- await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")
diff --git a/tests/fetchers/async/test_dynamic_session.py b/tests/fetchers/async/test_dynamic_session.py
deleted file mode 100644
index 9c8671a..0000000
--- a/tests/fetchers/async/test_dynamic_session.py
+++ /dev/null
@@ -1,139 +0,0 @@
-import pytest
-import asyncio
-
-import pytest_httpbin
-
-from scrapling.fetchers import AsyncDynamicSession
-
-
-@pytest_httpbin.use_class_based_httpbin
-class TestAsyncDynamicSession:
- """Test AsyncDynamicSession"""
-
- # The `AsyncDynamicSession` is inheriting from `DynamicSession` class so no need to repeat all the tests
- @pytest.fixture
- def urls(self, httpbin):
- return {
- "basic": f"{httpbin.url}/get",
- "html": f"{httpbin.url}/html",
- }
-
- async def test_concurrent_async_requests(self, urls):
- """Test concurrent requests with async session"""
- async with AsyncDynamicSession(max_pages=3) as session:
- # Launch multiple concurrent requests
- tasks = [
- session.fetch(urls["basic"]),
- session.fetch(urls["html"]),
- session.fetch(urls["basic"])
- ]
-
- assert session.max_pages == 3
- assert session.page_pool.max_pages == 3
- assert session.context is not None
-
- responses = await asyncio.gather(*tasks)
-
- # All should succeed
- assert all(r.status == 200 for r in responses)
-
- # Check pool stats
- stats = session.get_pool_stats()
- assert stats["total_pages"] <= 3
-
- # After exit, should be closed
- assert session._is_alive is False
-
- # Should raise RuntimeError when used after closing
- with pytest.raises(RuntimeError):
- await session.fetch(urls["basic"])
-
- async def test_page_pool_management(self, urls):
- """Test page pool creation and reuse"""
- async with AsyncDynamicSession() as session:
- # The first request creates a page, and it stays open afterwards
- response = await session.fetch(urls["basic"])
- assert response.status == 200
- assert session.page_pool.pages_count == 1
- page = session.page_pool.pages[0].page
- assert session.page_pool.pages[0].state == "ready"
-
- # The second request reuses the same tab
- response = await session.fetch(urls["html"])
- assert response.status == 200
- assert session.page_pool.pages_count == 1
- assert session.page_pool.pages[0].page is page
-
- # Check pool stats
- stats = session.get_pool_stats()
- assert stats["total_pages"] == 1
- assert stats["busy_pages"] == 0
- assert stats["max_pages"] == 1
-
- async def test_close_pages(self, urls):
- """Closing the tabs empties the pool, and the next request opens a fresh one"""
- async with AsyncDynamicSession() as session:
- await session.fetch(urls["basic"])
- page = session.page_pool.pages[0].page
-
- await session.close_pages()
- assert session.page_pool.pages_count == 0
- assert page.is_closed()
-
- response = await session.fetch(urls["html"])
- assert response.status == 200
- assert session.page_pool.pages_count == 1
- assert session.page_pool.pages[0].page is not page
-
- async def test_per_request_headers_do_not_leak_into_the_next_request(self, httpbin):
- """A reused tab gets its settings reset, so headers from the previous request are gone"""
- async with AsyncDynamicSession() as session:
- response = await session.fetch(f"{httpbin.url}/headers", extra_headers={"X-Scrapling-First": "1"})
- assert "X-Scrapling-First" in response.get_all_text()
-
- response = await session.fetch(f"{httpbin.url}/headers", extra_headers={"X-Scrapling-Second": "2"})
- assert "X-Scrapling-Second" in response.get_all_text()
- assert "X-Scrapling-First" not in response.get_all_text(), "the new headers must replace the old ones"
-
- response = await session.fetch(f"{httpbin.url}/headers")
- assert "X-Scrapling-Second" not in response.get_all_text()
- assert "X-Scrapling-First" not in response.get_all_text()
-
- async def test_resource_blocking_does_not_leak_into_the_next_request(self, httpbin):
- """Routes registered for one request are removed before the tab is reused"""
- js = """async () => {
- const img = document.createElement('img');
- const done = new Promise(r => { img.onload = () => r('loaded'); img.onerror = () => r('blocked'); });
- img.src = '/image/png?' + Math.random();
- document.body.appendChild(img);
- return await done;
- }"""
- results = []
-
- async def probe(page):
- results.append(await page.evaluate(js))
-
- async with AsyncDynamicSession() as session:
- await session.fetch(f"{httpbin.url}/html", disable_resources=True, page_action=probe)
- await session.fetch(f"{httpbin.url}/html", page_action=probe)
- assert results == ["blocked", "loaded"]
- assert session.page_pool.pages_count == 1
-
- async def test_dynamic_session_with_options(self, urls):
- """Test AsyncDynamicSession with various options"""
- async with AsyncDynamicSession(
- headless=False,
- disable_resources=True,
- extra_headers={"X-Test": "value"}
- ) as session:
- response = await session.fetch(urls["html"])
- assert response.status == 200
-
- async def test_error_handling_in_fetch(self, urls):
- """Test error handling during fetch"""
- async with AsyncDynamicSession() as session:
- # Test with invalid URL
- with pytest.raises(Exception):
- await session.fetch("invalid://url")
- assert session.page_pool.pages_count == 0, "errored tabs are closed and evicted"
diff --git a/tests/fetchers/async/test_stealth.py b/tests/fetchers/async/test_stealth.py
index bba39d8..3e0abfc 100644
--- a/tests/fetchers/async/test_stealth.py
+++ b/tests/fetchers/async/test_stealth.py
@@ -6,6 +6,7 @@ from scrapling import StealthyFetcher
StealthyFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin
@pytest.mark.asyncio
class TestStealthyFetcher:
@@ -23,7 +24,7 @@ class TestStealthyFetcher:
"basic_url": f"{url}/get",
"html_url": f"{url}/html",
"delayed_url": f"{url}/delay/10", # 10 Seconds delay response
- "cookies_url": f"{url}/cookies/set/test/value"
+ "cookies_url": f"{url}/cookies/set/test/value",
}
async def test_basic_fetch(self, fetcher, urls):
@@ -35,10 +36,11 @@ class TestStealthyFetcher:
async def test_cookies_loading(self, fetcher, urls):
"""Test if cookies are set after the request"""
response = await fetcher.async_fetch(urls["cookies_url"])
- cookies = {response.cookies[0]['name']: response.cookies[0]['value']}
+ cookies = {response.cookies[0]["name"]: response.cookies[0]["value"]}
assert cookies == {"test": "value"}
- async def test_automation(self, fetcher, urls):
+ @pytest.mark.parametrize("humanize", [False, True])
+ async def test_automation(self, fetcher, urls, humanize):
"""Test if automation breaks the code or not"""
async def scroll_page(page):
@@ -47,13 +49,12 @@ class TestStealthyFetcher:
await page.mouse.up()
return page
- assert (
- await fetcher.async_fetch(urls["html_url"], page_action=scroll_page, humanize=True)
- ).status == 200
+ assert (await fetcher.async_fetch(urls["html_url"], page_action=scroll_page, humanize=humanize)).status == 200
@pytest.mark.parametrize(
"kwargs",
[
+ {"real_chrome": True, "disable_resources": True},
{"block_webrtc": True, "allow_webgl": True},
{"block_webrtc": False, "allow_webgl": True},
{"block_webrtc": True, "allow_webgl": False, "disable_resources": True},
@@ -64,6 +65,9 @@ class TestStealthyFetcher:
"wait": 10,
"cookies": [{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],
"google_search": True,
+ "real_chrome": True,
+ "locale": "en-US",
+ "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0",
"extra_headers": {"ayo": ""},
"selector_config": {"keep_comments": False, "keep_cdata": False},
"additional_args": {},
@@ -72,8 +76,11 @@ class TestStealthyFetcher:
)
async def test_properties(self, fetcher, urls, kwargs):
"""Test if different arguments break the code or not"""
- response = await fetcher.async_fetch(
- urls["html_url"],
- **kwargs
- )
+ response = await fetcher.async_fetch(urls["html_url"], **kwargs)
assert response.status == 200
+
+ async def test_cdp_url_invalid(self, fetcher, urls):
+ with pytest.raises(TypeError):
+ await fetcher.async_fetch(urls["html_url"], cdp_url="blahblah")
+ with pytest.raises(Exception):
+ await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")
diff --git a/tests/fetchers/async/test_stealth_session.py b/tests/fetchers/async/test_stealth_session.py
index 15ea981..9035163 100644
--- a/tests/fetchers/async/test_stealth_session.py
+++ b/tests/fetchers/async/test_stealth_session.py
@@ -1,4 +1,3 @@
-
import pytest
import asyncio
@@ -7,6 +6,7 @@ import pytest_httpbin
from scrapling.fetchers import AsyncStealthySession
@pytest_httpbin.use_class_based_httpbin
@pytest.mark.asyncio
class TestAsyncStealthySession:
@@ -24,11 +24,7 @@ class TestAsyncStealthySession:
"""Test concurrent requests with async session"""
async with AsyncStealthySession(max_pages=3) as session:
# Launch multiple concurrent requests
- tasks = [
- session.fetch(urls["basic"]),
- session.fetch(urls["html"]),
- session.fetch(urls["basic"])
- ]
+ tasks = [session.fetch(urls["basic"]), session.fetch(urls["html"]), session.fetch(urls["basic"])]
assert session.max_pages == 3
assert session.page_pool.max_pages == 3
@@ -121,13 +117,16 @@ class TestAsyncStealthySession:
assert results == ["blocked", "loaded"]
assert session.page_pool.pages_count == 1
- async def test_stealthy_session_with_options(self, urls):
+ @pytest.mark.parametrize(
+ "options",
+ [
+ {"max_pages": 1, "block_webrtc": True, "allow_webgl": True},
+ {"headless": False, "disable_resources": True, "extra_headers": {"X-Test": "value"}},
+ ],
+ )
+ async def test_stealthy_session_with_options(self, urls, options):
"""Test AsyncStealthySession with various options"""
- async with AsyncStealthySession(
- max_pages=1,
- block_webrtc=True,
- allow_webgl=True
- ) as session:
+ async with AsyncStealthySession(**options) as session:
response = await session.fetch(urls["html"])
assert response.status == 200
diff --git a/tests/fetchers/sync/test_dynamic.py b/tests/fetchers/sync/test_stealth.py
similarity index 80%
rename from tests/fetchers/sync/test_dynamic.py
rename to tests/fetchers/sync/test_stealth.py
index cac8a73..8fcaf26 100644
--- a/tests/fetchers/sync/test_dynamic.py
+++ b/tests/fetchers/sync/test_stealth.py
@@ -1,17 +1,18 @@
import pytest
import pytest_httpbin
-from scrapling import DynamicFetcher
+from scrapling import StealthyFetcher
-DynamicFetcher.adaptive = True
+StealthyFetcher.adaptive = True
@pytest_httpbin.use_class_based_httpbin
-class TestDynamicFetcher:
+class TestStealthyFetcher:
@pytest.fixture(scope="class")
def fetcher(self):
"""Fixture to create a StealthyFetcher instance for the entire test class"""
- return DynamicFetcher
+ return StealthyFetcher
@pytest.fixture(autouse=True)
def setup_urls(self, httpbin):
@@ -21,20 +22,17 @@ class TestDynamicFetcher:
self.status_501 = f"{httpbin.url}/status/501"
self.basic_url = f"{httpbin.url}/get"
self.html_url = f"{httpbin.url}/html"
- self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response
+ self.delayed_url = f"{httpbin.url}/delay/10"
self.cookies_url = f"{httpbin.url}/cookies/set/test/value"
def test_basic_fetch(self, fetcher):
"""Test doing a basic fetch request with multiple statuses"""
assert fetcher.fetch(self.status_200).status == 200
- # There's a bug with playwright makes it crashes if a URL returns status code 4xx/5xx without body, let's disable this till they reply to my issue report
- # assert fetcher.fetch(self.status_404).status == 404
- # assert fetcher.fetch(self.status_501).status == 501
def test_cookies_loading(self, fetcher):
"""Test if cookies are set after the request"""
response = fetcher.fetch(self.cookies_url)
- cookies = {response.cookies[0]['name']: response.cookies[0]['value']}
+ cookies = {response.cookies[0]["name"]: response.cookies[0]["value"]}
assert cookies == {"test": "value"}
def test_automation(self, fetcher):
@@ -78,8 +76,5 @@ class TestDynamicFetcher:
with pytest.raises(TypeError):
fetcher.fetch(self.html_url, cdp_url="blahblah")
- with pytest.raises(TypeError):
- fetcher.fetch(self.html_url, cdp_url="blahblah")
-
with pytest.raises(Exception):
fetcher.fetch(self.html_url, cdp_url="ws://blahblah")
diff --git a/tests/fetchers/sync/test_stealth_session.py b/tests/fetchers/sync/test_stealth_session.py
index 5285b35..d60e5bd 100644
--- a/tests/fetchers/sync/test_stealth_session.py
+++ b/tests/fetchers/sync/test_stealth_session.py
@@ -3,7 +3,6 @@ import pytest
import pytest_httpbin
from scrapling.engines.constants import DEFAULT_ARGS, STEALTH_ARGS
-from scrapling.engines._browsers._controllers import DynamicSession
from scrapling.engines._browsers._stealth import StealthySession, __CF_PATTERN__
@@ -19,7 +18,7 @@ class TestStealthConstants:
test_urls = [
"https://challenges.cloudflare.com/cdn-cgi/challenge-platform/h/123456",
"https://challenges.cloudflare.com/cdn-cgi/challenge-platform/orchestrate/jsch/v1",
- "http://challenges.cloudflare.com/cdn-cgi/challenge-platform/scripts/abc"
+ "http://challenges.cloudflare.com/cdn-cgi/challenge-platform/scripts/abc",
]
for url in test_urls:
@@ -29,7 +28,7 @@ class TestStealthConstants:
non_matching_urls = [
"https://example.com/challenge",
"https://cloudflare.com/something",
- "https://challenges.cloudflare.com/other-path"
+ "https://challenges.cloudflare.com/other-path",
]
for url in non_matching_urls:
@@ -69,20 +68,12 @@ class TestLaunchFlags:
):
assert flag in with_extra, f"{flag} lost to extra_flags"
- def test_dynamic_session_keeps_its_defaults(self):
- """DynamicSession passes no bundle, so it never lost a flag - a control that
- held before this fix and pins the path the fix also touches."""
- session = DynamicSession(headless=True, extra_flags=[self.USER_FLAG])
- args = list(session._browser_options["args"])
-
- assert self.USER_FLAG in args
- assert set(DEFAULT_ARGS) <= set(args), "a default flag was dropped"
-
@pytest_httpbin.use_class_based_httpbin
class TestStealthySession:
-
"""All the code is tested in the async version tests, so no need to repeat it here. The async class inherits from this one."""
+
@pytest.fixture(autouse=True)
def setup_urls(self, httpbin):
"""Fixture to set up URLs for testing"""
@@ -105,7 +96,6 @@ class TestStealthySession:
timeout=60000,
cookies=[{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],
) as session:
-
assert session.max_pages == 1
assert session._config.headless is True
assert session._config.disable_resources is True
@@ -115,7 +105,7 @@ class TestStealthySession:
assert session.context is not None
# Test Cloudflare detection
- for cloudflare_type in ('managed', 'interactive', 'non-interactive'):
+ for cloudflare_type in ("managed", "interactive", "non-interactive"):
page_content = f"""
<html>
<script>
diff --git a/tests/fetchers/test_cloudflare_solver.py b/tests/fetchers/test_cloudflare_solver.py
index e9915c7..7184c4a 100644
--- a/tests/fetchers/test_cloudflare_solver.py
+++ b/tests/fetchers/test_cloudflare_solver.py
@@ -2,7 +2,6 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from patchright._impl._errors import Error as PatchrightError
-from playwright._impl._errors import Error as PlaywrightError
from scrapling.engines.toolbelt.convertor import ResponseFactory
from scrapling.engines._browsers._stealth import StealthySession, AsyncStealthySession, __CF_MAX_SOLVE_ATTEMPTS__
@@ -109,20 +108,18 @@ class TestAsyncSolver:
class TestPageContentRetries:
- @pytest.mark.parametrize("error_class", [PatchrightError, PlaywrightError])
- def test_sync_content_errors_are_retried(self, error_class):
- """Both patchright and playwright errors must trigger the retry workaround, not crash"""
+ def test_sync_content_errors_are_retried(self):
+ """Patchright errors must trigger the retry workaround"""
page = MagicMock()
- page.content.side_effect = [error_class("Page.content: page is navigating"), CLEAN_PAGE]
+ page.content.side_effect = [PatchrightError("Page.content: page is navigating"), CLEAN_PAGE]
assert ResponseFactory._get_page_content(page) == CLEAN_PAGE
assert page.wait_for_timeout.call_count == 1
@pytest.mark.asyncio
- @pytest.mark.parametrize("error_class", [PatchrightError, PlaywrightError])
- async def test_async_content_errors_are_retried(self, error_class):
+ async def test_async_content_errors_are_retried(self):
page = MagicMock()
- page.content = AsyncMock(side_effect=[error_class("Page.content: page is navigating"), CLEAN_PAGE])
+ page.content = AsyncMock(side_effect=[PatchrightError("Page.content: page is navigating"), CLEAN_PAGE])
page.wait_for_timeout = AsyncMock()
assert await ResponseFactory._get_async_page_content(page) == CLEAN_PAGE
@@ -154,11 +151,3 @@ class TestLocaleLaunchFlags:
"""Remote browsers can't take launch flags, so the context option is the best effort left"""
session = StealthySession(locale="fr-FR", cdp_url="ws://127.0.0.1:9222/devtools/browser/x")
assert session._context_options.get("locale") == "fr-FR"
-
- def test_dynamic_session_gets_the_same_flags(self):
- from scrapling.engines._browsers._controllers import DynamicSession
-
- session = DynamicSession(locale="en-GB")
- assert "--lang=en-GB" in session._browser_options["args"]
- assert "--accept-lang=en-GB,en" in session._browser_options["args"]
- assert "locale" not in session._context_options
diff --git a/tests/fetchers/test_network_browser.py b/tests/fetchers/test_network_browser.py
index 90ba4df..b42ed82 100644
--- a/tests/fetchers/test_network_browser.py
+++ b/tests/fetchers/test_network_browser.py
@@ -15,12 +15,10 @@ from urllib.parse import urlsplit
import pytest
from patchright.async_api import Response as AsyncPatchrightResponse
from patchright.sync_api import Response as PatchrightResponse
-from playwright.async_api import Response as AsyncPlaywrightResponse
-from playwright.sync_api import Response as PlaywrightResponse
from scrapling.core._types import Any, Generator
from scrapling.core.ai._network_formatting import _request_details
-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession
+from scrapling.fetchers import AsyncStealthySession, StealthySession
from scrapling.engines.toolbelt.custom import Response
from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
@@ -242,7 +240,7 @@ def _options(**kwargs: Any) -> dict[str, Any]:
}
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_network_disabled_by_default(session_type: Any) -> None:
session = session_type()
context = Mock()
@@ -255,7 +253,7 @@ def test_sync_network_disabled_by_default(session_type: Any) -> None:
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_network_disabled_by_default(session_type: Any) -> None:
session = session_type()
context = Mock()
@@ -338,7 +336,8 @@ def _check_later(network: Any, after_id: int) -> tuple[Any, Any, Any]:
return post[0], status[0], network.search(url_pattern=r"/api/large$")[0]
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:
with session_type(
**_options(
@@ -407,8 +406,9 @@ def test_sync_network_browser_history(session_type: Any, network_url: str, relea
assert post.headers["x-recorded"] == "response"
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:
async def initial_page(page: Any) -> None:
await page.locator("body[data-initial=ready]").wait_for(state="attached")
@@ -493,7 +493,8 @@ def _check_eviction(network: Any) -> int:
return last_id
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_network_browser_eviction_and_clear(session_type: Any, network_url: str) -> None:
with session_type(**_options(max_recorded_requests=2)) as session:
session.fetch(network_url + "/")
@@ -513,8 +514,9 @@ def test_sync_network_browser_eviction_and_clear(session_type: Any, network_url:
assert records[0].meta["request_body"] == records[0].body == b"fresh"
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_network_browser_eviction_and_clear(session_type: Any, network_url: str) -> None:
async with session_type(**_options(max_recorded_requests=2)) as session:
await session.fetch(network_url + "/")
@@ -537,7 +539,8 @@ async def test_async_network_browser_eviction_and_clear(session_type: Any, netwo
PROXY_HTML = '<html><body><script>fetch("/api").then(response => response.json()).then(value => document.body.dataset.api = JSON.stringify(value));</script></body></html>'
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_network_browser_temporary_proxy_context(session_type: Any) -> None:
def setup(page: Any) -> None:
page.route(
@@ -564,8 +567,9 @@ def test_sync_network_browser_temporary_proxy_context(session_type: Any) -> None
assert record.json() == {"temporary": True}
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_network_browser_temporary_proxy_context(session_type: Any) -> None:
async def setup(page: Any) -> None:
await page.route(
@@ -602,7 +606,8 @@ def _check_saved_override(network: Any, network_url: str) -> None:
assert not network.search(url_pattern=r"/api/original$")
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_network_browser_keeps_route_overrides_after_close(
session_type: Any, network_url: str, monkeypatch: Any
) -> None:
@@ -628,8 +633,9 @@ def test_sync_network_browser_keeps_route_overrides_after_close(
assert f"<GET {network_url}/>" in logger.info.call_args.args[0]
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_network_browser_keeps_route_overrides_after_close(
session_type: Any, network_url: str, monkeypatch: Any
) -> None:
@@ -655,7 +661,8 @@ async def test_async_network_browser_keeps_route_overrides_after_close(
assert f"<GET {network_url}/>" in logger.info.call_args.args[0]
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
@pytest.mark.parametrize("target", ["page", "context", "session"])
def test_sync_network_close_during_unfinished_response(
session_type: Any, target: str, network_url: str, network_gates: dict[str, _Gate]
@@ -689,8 +696,9 @@ def test_sync_network_close_during_unfinished_response(
assert saved.body == BLANK
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
@pytest.mark.parametrize("target", ["page", "context", "session"])
async def test_async_network_close_during_unfinished_response(
session_type: Any, target: str, network_url: str, network_gates: dict[str, _Gate]
@@ -737,7 +745,8 @@ def _check_burst(network: Any, cursor: int, path: str, expected_paths: list[str]
return newer[0].meta["network_id"]
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_network_burst_completion_order_and_pagination(
session_type: Any, network_url: str, network_gates: dict[str, _Gate]
) -> None:
@@ -775,8 +784,9 @@ def test_sync_network_burst_completion_order_and_pagination(
assert all(record.json() == {"ok": True} for record in records)
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_network_burst_completion_order_and_pagination(
session_type: Any, network_url: str, network_gates: dict[str, _Gate]
) -> None:
@@ -855,7 +865,8 @@ def _check_wire_bodies(
assert records[path].body == b"" and "body_note" not in records[path].meta
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_network_wire_bodies_remain_readable_after_close(session_type: Any, network_url: str) -> None:
with session_type(**_options()) as session:
session.fetch(network_url + "/blank")
@@ -873,8 +884,9 @@ def test_sync_network_wire_bodies_remain_readable_after_close(session_type: Any,
_check_wire_bodies(session.network, wire, native_bodies, native_texts)
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_network_wire_bodies_remain_readable_after_close(session_type: Any, network_url: str) -> None:
async with session_type(**_options()) as session:
await session.fetch(network_url + "/blank")
@@ -912,9 +924,8 @@ def _check_mime_bodies(network: Any, wire: dict[str, list[int]], reads: list[str
assert records[path].json() == {"saved": True}
- ("session_type", "response_type"), [(DynamicSession, PlaywrightResponse), (StealthySession, PatchrightResponse)]
-)
[email protected](("session_type", "response_type"), [(StealthySession, PatchrightResponse)])
def test_sync_network_reads_only_text_bodies(
session_type: Any, response_type: Any, network_url: str, monkeypatch: pytest.MonkeyPatch
) -> None:
@@ -938,10 +949,11 @@ def test_sync_network_reads_only_text_bodies(
_check_mime_bodies(session.network, wire, reads)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("session_type", "response_type"),
- [(AsyncDynamicSession, AsyncPlaywrightResponse), (AsyncStealthySession, AsyncPatchrightResponse)],
+ [(AsyncStealthySession, AsyncPatchrightResponse)],
)
async def test_async_network_reads_only_text_bodies(
session_type: Any, response_type: Any, network_url: str, monkeypatch: pytest.MonkeyPatch
@@ -993,7 +1005,8 @@ def _check_unlimited_search(network: Any, after_id: int) -> None:
assert len(network.search(limit=None)) == 125 and len(network.search()) == 100
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_network_unlimited_search_respects_retention(session_type: Any, network_url: str) -> None:
with session_type(**_options(max_recorded_requests=125)) as session:
session.fetch(network_url + "/blank")
@@ -1008,8 +1021,9 @@ def test_sync_network_unlimited_search_respects_retention(session_type: Any, net
_check_unlimited_search(session.network, after_id)
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_network_unlimited_search_respects_retention(session_type: Any, network_url: str) -> None:
async with session_type(**_options(max_recorded_requests=125)) as session:
await session.fetch(network_url + "/blank")
diff --git a/tests/fetchers/test_network_config.py b/tests/fetchers/test_network_config.py
index 585afe2..dc2ba82 100644
--- a/tests/fetchers/test_network_config.py
+++ b/tests/fetchers/test_network_config.py
@@ -2,7 +2,7 @@ import pytest
from scrapling.core._types import Any
from scrapling.engines._browsers._validators import PlaywrightConfig, StealthConfig, validate
-from scrapling.fetchers import DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession
+from scrapling.fetchers import StealthySession, AsyncStealthySession
@pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])
@@ -21,7 +21,7 @@ def test_network_config_invalid(model: type[PlaywrightConfig], values: dict[str,
validate(values, model)
[email protected]("session_type", [DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [StealthySession, AsyncStealthySession])
@pytest.mark.parametrize("enabled", [False, True])
def test_sessions_configure_one_network_history(session_type: Any, enabled: bool) -> None:
session = session_type(record_requests=enabled, max_recorded_requests=7)
diff --git a/tests/fetchers/test_public_api.py b/tests/fetchers/test_public_api.py
new file mode 100644
index 0000000..c2fd9c6
--- /dev/null
+++ b/tests/fetchers/test_public_api.py
@@ -0,0 +1,51 @@
+from importlib import import_module
+from importlib.machinery import ModuleSpec
+from json import dumps
+from pathlib import Path
+
+import pytest
+
+from scrapling.engines.toolbelt import fingerprints
+
+
[email protected]("module", ["scrapling", "scrapling.fetchers"])
[email protected]("name", ["DynamicFetcher", "DynamicSession", "AsyncDynamicSession", "PlayWrightFetcher"])
+def test_dynamic_imports_are_removed(module: str, name: str) -> None:
+ package = import_module(module)
+ assert name not in package.__all__
+ with pytest.raises(ImportError):
+ exec(f"from {module} import {name}")
+
+
[email protected]("module", ["scrapling.fetchers.chrome", "scrapling.engines._browsers._controllers"])
+def test_dynamic_modules_are_removed(module: str) -> None:
+ with pytest.raises(ModuleNotFoundError):
+ import_module(module)
+
+
[email protected]("browser_mode", [True, "chrome"])
+def test_browser_version_uses_patchright_when_playwright_is_installed(
+ browser_mode: bool | str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ packages: dict[str, ModuleSpec] = {}
+ looked_up: list[str] = []
+ for name, version in (("patchright", "998.0.0.0"), ("playwright", "999.0.0.0")):
+ directory = tmp_path / name
+ bundle = directory / "driver" / "package"
+ bundle.mkdir(parents=True)
+ (bundle / "browsers.json").write_text(dumps({"browsers": [{"name": "chromium", "browserVersion": version}]}))
+ packages[name] = ModuleSpec(name, None, origin=str(directory / "__init__.py"))
+
+ def find_spec(name: str) -> ModuleSpec:
+ looked_up.append(name)
+ return packages[name]
+
+ monkeypatch.setattr(fingerprints, "find_spec", find_spec)
+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "macos")
+ fingerprints.driven_browser_version.cache_clear()
+ try:
+ assert fingerprints.driven_browser_version() == 998
+ assert "Chrome/998.0.0.0" in fingerprints.generate_headers(browser_mode=browser_mode)["User-Agent"]
+ assert looked_up == ["patchright"]
+ finally:
+ fingerprints.driven_browser_version.cache_clear()
diff --git a/tests/fetchers/test_session_state.py b/tests/fetchers/test_session_state.py
index 3bb5fce..48a1aa2 100644
--- a/tests/fetchers/test_session_state.py
+++ b/tests/fetchers/test_session_state.py
@@ -6,7 +6,7 @@ import pytest
from scrapling.core._types import Any
from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession
+from scrapling.fetchers import AsyncStealthySession, StealthySession
URL = "http://state.test/"
@@ -82,7 +82,8 @@ def _check_json(path: Path, native_state: dict[str, Any]) -> None:
assert "sessionStorage" not in origin
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_state_round_trip(session_type: Any, tmp_path: Path) -> None:
path, native_path = tmp_path / "state.json", tmp_path / "native.json"
with session_type(**_options()) as source:
@@ -114,8 +115,9 @@ def test_sync_state_round_trip(session_type: Any, tmp_path: Path) -> None:
_check_json(path, loads(native_path.read_text()))
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_state_round_trip(session_type: Any, tmp_path: Path) -> None:
async def route_page(route: Any) -> None:
await route.fulfill(body=HTML, content_type="text/html")
@@ -150,7 +152,8 @@ async def test_async_state_round_trip(session_type: Any, tmp_path: Path) -> None
_check_json(path, loads(native_path.read_text()))
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_state_file_errors(session_type: Any, tmp_path: Path) -> None:
invalid = tmp_path / "invalid.json"
invalid.write_text("{")
@@ -166,8 +169,9 @@ def test_sync_state_file_errors(session_type: Any, tmp_path: Path) -> None:
session.save_state("")
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_state_file_errors(session_type: Any, tmp_path: Path) -> None:
invalid = tmp_path / "invalid.json"
invalid.write_text("{")
@@ -183,7 +187,7 @@ async def test_async_state_file_errors(session_type: Any, tmp_path: Path) -> Non
await session.save_state("")
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_state_before_start(session_type: Any, tmp_path: Path) -> None:
session = session_type(**_options())
for method in (session.save_state, session.load_state):
@@ -192,7 +196,7 @@ def test_sync_state_before_start(session_type: Any, tmp_path: Path) -> None:
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_state_before_start(session_type: Any, tmp_path: Path) -> None:
session = session_type(**_options())
for method in (session.save_state, session.load_state):
@@ -200,7 +204,8 @@ async def test_async_state_before_start(session_type: Any, tmp_path: Path) -> No
await method(tmp_path / "state.json")
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_state_with_proxy_rotation(session_type: Any, tmp_path: Path) -> None:
with session_type(**_options(proxy_rotator=ProxyRotator(["http://127.0.0.1:1"]))) as session:
for method in (session.save_state, session.load_state):
@@ -208,8 +213,9 @@ def test_sync_state_with_proxy_rotation(session_type: Any, tmp_path: Path) -> No
method(tmp_path / "state.json")
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_state_with_proxy_rotation(session_type: Any, tmp_path: Path) -> None:
async with session_type(**_options(proxy_rotator=ProxyRotator(["http://127.0.0.1:1"]))) as session:
for method in (session.save_state, session.load_state):
diff --git a/tests/fetchers/test_shadow_browser.py b/tests/fetchers/test_shadow_browser.py
index 2a543d7..3e7c2a8 100644
--- a/tests/fetchers/test_shadow_browser.py
+++ b/tests/fetchers/test_shadow_browser.py
@@ -4,7 +4,7 @@ import pytest
from scrapling.core._types import Any
from scrapling.engines.toolbelt.custom import Response
-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession
+from scrapling.fetchers import AsyncStealthySession, StealthySession
HTML = """<!DOCTYPE html><html><head><title>Shadow test</title></head><body>
@@ -48,7 +48,8 @@ async def setup_async_page(page: Any) -> None:
await page.route("**/*", lambda route: route.fulfill(status=200, content_type="text/html", body=HTML))
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_shadow_browser(session_type: Any) -> None:
with session_type(
executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"),
@@ -60,8 +61,9 @@ def test_sync_shadow_browser(session_type: Any) -> None:
assert not session.fetch("https://shadow.test/", pierce_shadow=False).css("#shadow")
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_shadow_browser(session_type: Any) -> None:
async with session_type(
executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"),
diff --git a/tests/fetchers/test_shadow_hidden_slots.py b/tests/fetchers/test_shadow_hidden_slots.py
index 633724d..0eb9b30 100644
--- a/tests/fetchers/test_shadow_hidden_slots.py
+++ b/tests/fetchers/test_shadow_hidden_slots.py
@@ -25,6 +25,7 @@ def check_clean_output(response: Response, visible: tuple[str, ...], hidden: tup
assert all(value not in output for value in hidden)
@pytest.mark.asyncio
@pytest.mark.parametrize(("attrs", "key", "value"), HIDDEN_ATTRIBUTES)
async def test_hidden_slots_keep_attributes_and_assigned_content(attrs: str, key: str, value: str) -> None:
@@ -66,6 +67,7 @@ async def test_hidden_slots_keep_attributes_and_assigned_content(attrs: str, key
assert response.body.decode() == body
@pytest.mark.asyncio
@pytest.mark.parametrize("assigned", [False, True])
@pytest.mark.parametrize(("attrs", "key", "value"), HIDDEN_ATTRIBUTES)
diff --git a/tests/fetchers/test_shadow_order.py b/tests/fetchers/test_shadow_order.py
index 95b0f87..cf0beda 100644
--- a/tests/fetchers/test_shadow_order.py
+++ b/tests/fetchers/test_shadow_order.py
@@ -2,7 +2,7 @@ from os import getenv
from pathlib import Path
import pytest
-from playwright.async_api import async_playwright
+from patchright.async_api import async_playwright
from scrapling.engines.toolbelt.convertor import ResponseFactory
from scrapling.engines.toolbelt.custom import Response
@@ -42,6 +42,7 @@ async def extract(html: str, setup: str) -> Response:
await browser.close()
@pytest.mark.asyncio
async def test_shadow_slots_follow_composed_order(tmp_path: Path) -> None:
response = await extract(
@@ -81,6 +82,7 @@ async def test_shadow_slots_follow_composed_order(tmp_path: Path) -> None:
assert not restored.css("#unused")
@pytest.mark.asyncio
@pytest.mark.parametrize("assigned", [False, True])
async def test_shadow_forwarded_slots_and_nested_hosts(assigned: bool) -> None:
diff --git a/tests/fetchers/test_shadow_roundtrip.py b/tests/fetchers/test_shadow_roundtrip.py
index c24250b..55a207a 100644
--- a/tests/fetchers/test_shadow_roundtrip.py
+++ b/tests/fetchers/test_shadow_roundtrip.py
@@ -1,7 +1,7 @@
from os import getenv
import pytest
-from playwright.async_api import async_playwright
+from patchright.async_api import async_playwright
from scrapling.core._types import extraction_types
from scrapling.core.shell import Convertor
@@ -26,6 +26,7 @@ def check_roundtrip(content: bytes) -> None:
assert response.body == content
@pytest.mark.asyncio
async def test_shadow_wrappers_roundtrip_across_host_tags() -> None:
async with async_playwright() as driver:
diff --git a/tests/fetchers/test_shadow_snapshot.py b/tests/fetchers/test_shadow_snapshot.py
index c2d5da2..b19ca05 100644
--- a/tests/fetchers/test_shadow_snapshot.py
+++ b/tests/fetchers/test_shadow_snapshot.py
@@ -1,11 +1,12 @@
from os import getenv
import pytest
-from playwright.async_api import async_playwright
+from patchright.async_api import async_playwright
from scrapling.engines.toolbelt.convertor import SHADOW_SNAPSHOT_JS, ResponseFactory
@pytest.mark.asyncio
async def test_shadow_snapshot_preserves_source_content() -> None:
async with async_playwright() as driver:
@@ -25,7 +26,8 @@ async def test_shadow_snapshot_preserves_source_content() -> None:
baseline = await page.content()
plain = await ResponseFactory.from_async_playwright_response(page, first, None, {}, pierce_shadow=True)
assert plain.body.decode() == baseline
- await page.evaluate("""() => {
+ await page.evaluate(
+ """() => {
const root = document.querySelector('#host').attachShadow({mode: 'open'});
root.innerHTML = '<div id="inside">SHADOW<slot name="label">WRONG-FALLBACK</slot><slot name="missing">USED-FALLBACK</slot><div id="nested"></div><svg viewBox="0 0 10 10"><path d="M0 0"/></svg></div>';
let host = root.querySelector('#nested');
@@ -73,17 +75,19 @@ async def test_shadow_snapshot_preserves_source_content() -> None:
root.append(scopedHost);
scopedHost.attachShadow({mode: 'open', customElementRegistry: registry}).innerHTML = '<shadow-scoped-counter>SCOPED-VALUE</shadow-scoped-counter>';
}
- }""")
+ }""",
+ isolated_context=False,
+ )
before = await page.content()
shadow_before = await page.locator("#host").evaluate("node => node.shadowRoot.innerHTML")
- count = await page.evaluate("globalThis.shadowConstructed")
+ count = await page.evaluate("globalThis.shadowConstructed", isolated_context=False)
noscript = await page.locator("#noscript").evaluate("node => node.outerHTML")
response = await ResponseFactory.from_async_playwright_response(
page, first, None, {"keep_comments": True}, pierce_shadow=True
)
assert await page.content() == before
assert await page.locator("#host").evaluate("node => node.shadowRoot.innerHTML") == shadow_before
- assert await page.evaluate("globalThis.shadowConstructed") == count
+ assert await page.evaluate("globalThis.shadowConstructed", isolated_context=False) == count
assert response.css("#host > shadow-root > #inside").get()
assert all(response.css(f"#level-{i}::text").get() == f"LEVEL-{i}" for i in range(4))
assert response.css("#data::text").get() == '{"name":"A&B<C>","title":"Bürger"}'
@@ -106,7 +110,7 @@ async def test_shadow_snapshot_preserves_source_content() -> None:
assert "LEVEL-3" in markdown
assert "INACTIVE" not in markdown
assert "FAKE-SHADOW" not in markdown
- if await page.evaluate("typeof CustomElementRegistry === 'function'"):
+ if await page.evaluate("typeof CustomElementRegistry === 'function'", isolated_context=False):
assert response.css("shadow-scoped-counter::text").get() == "SCOPED-VALUE"
await page.set_content(
'<!DOCTYPE html><html><body><template id="only"><div></div></template></body></html>'
diff --git a/tests/fetchers/test_snapshot.py b/tests/fetchers/test_snapshot.py
index 0e520a3..a05de63 100644
--- a/tests/fetchers/test_snapshot.py
+++ b/tests/fetchers/test_snapshot.py
@@ -5,7 +5,7 @@ from unittest.mock import AsyncMock, Mock
import pytest
from scrapling.core._types import Any
-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession
+from scrapling.fetchers import AsyncStealthySession, StealthySession
HTML = """<!DOCTYPE html><html><body>
@@ -15,7 +15,7 @@ HTML = """<!DOCTYPE html><html><body>
</body></html>"""
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
@pytest.mark.parametrize("css_selector", [None, "main"])
def test_sync_snapshot_forwards_options_and_errors(session_type: Any, css_selector: str | None) -> None:
page = Mock()
@@ -34,7 +34,7 @@ def test_sync_snapshot_forwards_options_and_errors(session_type: Any, css_select
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
@pytest.mark.parametrize("css_selector", [None, "main"])
async def test_async_snapshot_forwards_options_and_errors(session_type: Any, css_selector: str | None) -> None:
page = Mock()
@@ -61,7 +61,8 @@ def _button_ref(snapshot: str) -> str:
return match[1]
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("session_type", [StealthySession])
def test_sync_snapshot_live_state_and_refs(session_type: Any) -> None:
def setup(page: Any) -> None:
page.route("**/*", lambda route: route.fulfill(status=200, content_type="text/html", body=HTML))
@@ -83,8 +84,9 @@ def test_sync_snapshot_live_state_and_refs(session_type: Any) -> None:
assert session.page_pool.pages_count == 1
@pytest.mark.asyncio
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("session_type", [AsyncStealthySession])
async def test_async_snapshot_live_state_and_refs(session_type: Any) -> None:
async def setup(page: Any) -> None:
await page.route("**/*", lambda route: route.fulfill(status=200, content_type="text/html", body=HTML))
diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py
index 8f36672..14e125f 100644
--- a/tests/fetchers/test_utils.py
+++ b/tests/fetchers/test_utils.py
@@ -260,7 +260,7 @@ class TestFingerprintFunctions:
def test_driven_browser_version(self):
"""Test that the driven Chromium version is read from the installed automation package"""
- assert isinstance(driven_browser_version("playwright"), int)
+ assert isinstance(driven_browser_version(), int)
assert driven_browser_version("not_a_real_package") is None
@pytest.mark.parametrize("browser_mode", [True, "chrome"])
diff --git a/tests/fetchers/test_validator.py b/tests/fetchers/test_validator.py
index 3827194..71e8068 100644
--- a/tests/fetchers/test_validator.py
+++ b/tests/fetchers/test_validator.py
@@ -1,7 +1,7 @@
import pytest
from types import SimpleNamespace
-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession
+from scrapling.fetchers import AsyncStealthySession, StealthySession
from scrapling.engines._browsers._validators import (
validate,
validate_fetch,
@@ -37,9 +37,7 @@ class TestValidators:
assert params.timeout == config.timeout
assert not hasattr(params, "unknown_option")
- @pytest.mark.parametrize(
- "session_type", [DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession]
- )
+ @pytest.mark.parametrize("session_type", [StealthySession, AsyncStealthySession])
def test_removed_xhr_option_is_rejected_by_sessions(self, session_type):
with pytest.raises(TypeError, match="capture_xhr was removed"):
session_type(capture_xhr=".*")
diff --git a/tests/spiders/test_browser_session.py b/tests/spiders/test_browser_session.py
new file mode 100644
index 0000000..df32a79
--- /dev/null
+++ b/tests/spiders/test_browser_session.py
@@ -0,0 +1,29 @@
+from os import getenv
+
+import pytest
+
+from scrapling.core._types import Any
+from scrapling.fetchers import AsyncStealthySession
+from scrapling.spiders import Request, SessionManager
+
+
[email protected]("lazy", [False, True])
+async def test_stealthy_browser_session_dispatch(lazy: bool) -> None:
+ async def setup(page: Any) -> None:
+ await page.route("**/*", lambda route: route.fulfill(content_type="text/html", body="<h1>Browser</h1>"))
+
+ session = AsyncStealthySession(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"), google_search=False)
+ manager = SessionManager().add("browser", session, lazy=lazy)
+ request = Request("https://spider.test/", sid="browser", page_setup=setup, meta={"source": "browser"})
+ async with manager:
+ assert session._is_alive is not lazy
+ response = await manager.fetch(request)
+ assert response.status == 200
+ assert response.css("h1::text").get() == "Browser"
+ assert response.request is request
+ assert response.meta["source"] == "browser"
+ assert session._is_alive
+ assert not session._is_alive
+ assert session.page_pool.pages_count == 0
diff --git a/tox.ini b/tox.ini
index 0e91178..7ca4e6a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -16,14 +16,14 @@ deps =
extras = ai,shell
commands =
# Run browser tests without parallelization (avoid browser conflicts)
- pytest --cov=scrapling --cov-report=xml -k "DynamicFetcher or StealthyFetcher" --verbose
+ pytest --cov=scrapling --cov-report=xml -m "browser" --verbose
# Run asyncio tests without parallelization (avoid GitHub CI nested loop issues)
- pytest --cov=scrapling --cov-report=xml -m "asyncio" -k "not (DynamicFetcher or StealthyFetcher)" --verbose --cov-append
+ pytest --cov=scrapling --cov-report=xml -m "asyncio and not browser" --verbose --cov-append
# Run everything else with parallelization (for speed)
- pytest --cov=scrapling --cov-report=xml -m "not asyncio" -k "not (DynamicFetcher or StealthyFetcher)" -n auto --cov-append
+ pytest --cov=scrapling --cov-report=xml -m "not asyncio and not browser" -n auto --cov-append
[testenv:pre-commit]
basepython = python3
deps = pre-commit
commands = pre-commit run --all-files --show-diff-on-failure
-skip_install = true
\ No newline at end of file
+skip_install = true