🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
test: update tests accordingly
Karim shoair335beb932aeb
diff --git a/.bandit.yml b/.bandit.ymlindex bd06507..d987c18 100644--- a/.bandit.yml+++ b/.bandit.yml@@ -8,4 +8,5 @@ skips:- B110 # Try, Except, Pass detected.- B104 # Possible binding to all interfaces.- B301 # Pickle and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue.-- B108 # Probable insecure usage of temp file/directory.\ No newline at end of file+- B108 # Probable insecure usage of temp file/directory.+- B102 # Use of exec detected.\ No newline at end of filediff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex ec5342c..2684042 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -111,7 +111,7 @@ pip3 install git+https://github.com/D4Vinci/Scrapling.git@dev## TestsScrapling includes a comprehensive test suite you can run with pytest. First, install all libraries and `pytest-plugins` listed in `tests/requirements.txt`. Then, running the tests will result in an output like this:```bash- $ pytest tests -n auto+ $ pytest tests -m "not asyncio and not browser" -n auto=============================== test session starts ===============================platform darwin -- Python 3.13.8, pytest-8.4.2, pluggy-1.6.0 -- /Users/<redacted>/.venv/bin/python3.13cachedir: .pytest_cache@@ -128,13 +128,14 @@ Scrapling includes a comprehensive test suite you can run with pytest. First, in```Here, `-n auto` runs tests in parallel across multiple processes to increase speed.-**Note:** You may need to run browser tests sequentially (`DynamicFetcher`/`StealthyFetcher`) to avoid conflicts. To run non-browser tests in parallel and browser tests separately:+**Note:** Run browser and async tests sequentially to avoid conflicts. To run the other tests in parallel and these groups separately:```bash# Non-browser tests (parallel)-pytest tests/ -k "not (DynamicFetcher or StealthyFetcher)" -n auto+pytest tests/ -m "not asyncio and not browser" -n auto# Browser tests (sequential)-pytest tests/ -k "DynamicFetcher or StealthyFetcher"+pytest tests/ -m "browser"+pytest tests/ -m "asyncio and not browser"```Bonus: You can also see the test coverage with the `pytest` plugin belowdiff --git a/pytest.ini b/pytest.iniindex 083e39a..b5914b6 100644--- a/pytest.ini+++ b/pytest.ini@@ -4,4 +4,5 @@ asyncio_default_fixture_loop_scope = functionaddopts = -p no:warnings --doctest-modules --ignore=setup.py --verbosemarkers =asyncio: marks tests as async+ browser: tests that start a real browserasyncio_fixture_scope = function\ No newline at end of filediff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.pyindex c3161da..a0654e5 100644--- a/tests/ai/test_ai_mcp.py+++ b/tests/ai/test_ai_mcp.py@@ -169,6 +169,7 @@ class TestOneShotTools:assert len(sessions) == 4 and all(not session._is_alive for session in sessions)assert not server._sessions+ @pytest.mark.browser@pytest.mark.asyncioasync def test_browser_fetch_once_renders_and_closes_through_mcp(self, monkeypatch):sessions: list[AsyncStealthySession] = []@@ -409,6 +410,7 @@ class TestSessionManagement:def server(self):return ScraplingMCPServer()+ @pytest.mark.browser@pytest.mark.asyncioasync def test_open_and_close_session(self, server):"""Test opening and closing a stealthy session"""@@ -423,6 +425,7 @@ class TestSessionManagement:assert isinstance(closed, SessionClosedModel)assert closed.session_id == session_id+ @pytest.mark.browser@pytest.mark.asyncioasync def test_list_sessions(self, server):"""Test listing sessions"""@@ -445,6 +448,7 @@ class TestSessionManagement:# Cleanupawait server.close_session(session_id)+ @pytest.mark.browser@pytest.mark.asyncioasync def test_browser_fetch_reuses_the_session(self, server, test_url):"""Test fetching a page twice through a persistent stealthy session"""@@ -462,6 +466,7 @@ class TestSessionManagement:await server.close_session(session_id)+ @pytest.mark.browser@pytest.mark.asyncioasync def test_browser_fetch_accepts_per_request_overrides(self, server, test_url):"""A per-request override is honored on a session fetch"""@@ -486,6 +491,7 @@ class TestSessionManagement:with pytest.raises(ValueError, match="not found"):await server.browser_fetch(url=test_url, session_id="nonexistent")+ @pytest.mark.browser@pytest.mark.asyncioasync def test_browser_fetch_with_closed_session(self, server, test_url):"""Test fetching with a session that has been closed"""@@ -496,6 +502,7 @@ class TestSessionManagement:with pytest.raises(ValueError, match="not found"):await server.browser_fetch(url=test_url, session_id=session_id)+ @pytest.mark.browser@pytest.mark.asyncioasync def test_browser_open_with_custom_id(self, server):"""Test opening a session with a custom session_id"""@@ -505,6 +512,7 @@ class TestSessionManagement:await server.close_session("my-session")+ @pytest.mark.browser@pytest.mark.asyncioasync def test_browser_open_duplicate_id_raises(self, server):"""Test that opening a session with a duplicate session_id raises an error"""@@ -1058,6 +1066,7 @@ class TestScreenshot:def server(self):return ScraplingMCPServer()+ @pytest.mark.browser@pytest.mark.asyncioasync def test_screenshot_png_with_stealthy_session(self, server, test_url):"""PNG screenshot via a stealthy session returns image and url content blocks"""@@ -1074,6 +1083,7 @@ class TestScreenshot:finally:await server.close_session(opened.session_id)+ @pytest.mark.browser@pytest.mark.asyncioasync def test_screenshot_jpeg_with_quality(self, server, test_url):"""JPEG screenshot with quality parameter via a stealthy session"""@@ -1088,6 +1098,7 @@ class TestScreenshot:finally:await server.close_session(opened.session_id)+ @pytest.mark.browser@pytest.mark.asyncioasync def test_screenshot_full_page_taller_than_viewport(self, server):"""full_page=True produces an image taller than the viewport-only capture"""@@ -1112,6 +1123,7 @@ class TestScreenshot:with pytest.raises(ValueError, match="not found"):await server.browser_screenshot(session_id="does-not-exist")+ @pytest.mark.browser@pytest.mark.asyncioasync def test_screenshot_quality_with_png_raises(self, server, test_url):"""quality is rejected when image_type is png"""diff --git a/tests/ai/test_browser_actions.py b/tests/ai/test_browser_actions.pyindex 265b8ae..ec41487 100644--- a/tests/ai/test_browser_actions.py+++ b/tests/ai/test_browser_actions.py@@ -129,6 +129,7 @@ async def test_invalid_later_action_prevents_other_action_kinds(invalid: dict[stassert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncio@pytest.mark.parametrize("slowly", [False, True])async def test_live_mixed_chain_resolves_targets_created_by_earlier_actions(slowly: bool) -> None:@@ -188,6 +189,7 @@ async def test_live_mixed_chain_resolves_targets_created_by_earlier_actions(slowassert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncio@pytest.mark.parametrize("kind", ["textbox", "move", "click", "wait_element"])async def test_live_stale_ref_stops_mixed_chain_without_retargeting(kind: str) -> None:@@ -224,6 +226,7 @@ async def test_live_stale_ref_stops_mixed_chain_without_retargeting(kind: str) -assert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncio@pytest.mark.parametrize("use_ref", [False, True])async def test_live_targets_keep_explicit_refs_and_ref_shaped_selectors_distinct(use_ref: bool) -> None:diff --git a/tests/ai/test_browser_dialog_live.py b/tests/ai/test_browser_dialog_live.pyindex fa95af4..8e3fc3d 100644--- a/tests/ai/test_browser_dialog_live.py+++ b/tests/ai/test_browser_dialog_live.py@@ -8,6 +8,7 @@ from scrapling.core._types import Anyfrom tests.ai.test_browser_wait import _browser@pytest.mark.asyncio@pytest.mark.parametrize("expression, reply, expected",@@ -50,6 +51,7 @@ async def test_dialog_live_native_results(expression: str, reply: dict[str, Any]assert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncio@pytest.mark.parametrize("trigger", ["selector", "coordinates", "keyboard"])async def test_dialog_live_queued_replies_and_unexpected_dialog(trigger: str) -> None:@@ -83,6 +85,7 @@ async def test_dialog_live_queued_replies_and_unexpected_dialog(trigger: str) ->assert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncio@pytest.mark.parametrize("slowly", [False, True])async def test_dialog_live_new_reply_after_previous_trigger(slowly: bool) -> None:@@ -112,6 +115,7 @@ async def test_dialog_live_new_reply_after_previous_trigger(slowly: bool) -> Nonassert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncioasync def test_dialog_live_keyboard_trigger() -> None:async with _browser() as (client, session, page):@@ -136,6 +140,7 @@ async def test_dialog_live_keyboard_trigger() -> None:assert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncioasync def test_dialog_live_delayed_trigger_during_wait() -> None:async with _browser() as (client, session, page):@@ -162,6 +167,7 @@ async def test_dialog_live_delayed_trigger_during_wait() -> None:assert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncio@pytest.mark.parametrize("accept", [False, True])async def test_dialog_live_beforeunload_navigation(accept: bool) -> None:@@ -199,6 +205,7 @@ async def test_dialog_live_beforeunload_navigation(accept: bool) -> None:assert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncio@pytest.mark.parametrize("fail", [False, True])async def test_dialog_live_unused_replies_do_not_reach_later_calls(fail: bool) -> None:@@ -223,6 +230,7 @@ async def test_dialog_live_unused_replies_do_not_reach_later_calls(fail: bool) -assert loads(await page.get_attribute("body", "data-result")) is False@pytest.mark.asyncioasync def test_dialog_live_cancel_removes_unused_reply() -> None:async with _browser() as (client, session, page):diff --git a/tests/ai/test_browser_evaluate.py b/tests/ai/test_browser_evaluate.pyindex d44d535..80f7338 100644--- a/tests/ai/test_browser_evaluate.py+++ b/tests/ai/test_browser_evaluate.py@@ -249,6 +249,7 @@ async def test_browser_evaluate_reserves_page_until_cancelled(cancel_mode: str)page.close.assert_not_called()@pytest.mark.asyncioasync def test_browser_evaluate_live_expressions_promises_context_and_page_state() -> None:server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))diff --git a/tests/ai/test_browser_extract.py b/tests/ai/test_browser_extract.pyindex 6965f51..3b41d12 100644--- a/tests/ai/test_browser_extract.py+++ b/tests/ai/test_browser_extract.py@@ -171,6 +171,7 @@ async def test_browser_extract_releases_the_page_after_failure() -> None:assert await server.browser_extract("browser") == SNAPSHOT@pytest.mark.asyncio@pytest.mark.parametrize("css_selector", [None, "main"])async def test_browser_extract_live_mcp_round_trip(css_selector: str | None) -> None:@@ -527,6 +528,7 @@ async def test_browser_extract_search_errors_are_value_errors(kwargs: dict[str,assert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncioasync def test_browser_extract_search_live_refs_and_state() -> None:server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))diff --git a/tests/ai/test_browser_extract_content.py b/tests/ai/test_browser_extract_content.pyindex 0ee78b3..4653f7a 100644--- a/tests/ai/test_browser_extract_content.py+++ b/tests/ai/test_browser_extract_content.py@@ -244,6 +244,7 @@ async def test_browser_extract_content_cancellation_releases_page(target: str |page.close.assert_not_called()@pytest.mark.asyncioasync def test_browser_extract_content_live_formats_preserve_current_page() -> None:async with _browser() as (client, session, page):@@ -300,6 +301,7 @@ async def test_browser_extract_content_live_formats_preserve_current_page() -> Nassert before["scroll"] == 500 and not navigations and not requests@pytest.mark.asyncioasync def test_browser_extract_content_live_selectors_refs_and_snapshot_scope() -> None:async with _browser() as (client, session, page):@@ -327,6 +329,7 @@ async def test_browser_extract_content_live_selectors_refs_and_snapshot_scope()assert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncioasync def test_browser_extract_content_live_target_errors_allow_next_call() -> None:async with _browser() as (client, session, page):diff --git a/tests/ai/test_browser_fill_fields.py b/tests/ai/test_browser_fill_fields.pyindex 14e1f50..a195182 100644--- a/tests/ai/test_browser_fill_fields.py+++ b/tests/ai/test_browser_fill_fields.py@@ -510,6 +510,7 @@ async def _browser() -> AsyncGenerator[tuple[Client, Any, Any], None]:assert not closed.is_error@pytest.mark.asyncioasync def test_browser_actions_fields_live_mixed_fields_and_clearing() -> None:async with _browser() as (client, session, page):@@ -580,6 +581,7 @@ async def test_browser_actions_fields_live_mixed_fields_and_clearing() -> None:assert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncioasync def test_browser_actions_fields_live_slow_typing_and_field_delays() -> None:async with _browser() as (client, session, page):@@ -634,6 +636,7 @@ async def test_browser_actions_fields_live_slow_typing_and_field_delays() -> Nonassert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncio@pytest.mark.parametrize("slowly", [False, True])async def test_browser_actions_fields_live_caret_selection_and_submit(slowly: bool) -> None:@@ -681,6 +684,7 @@ async def test_browser_actions_fields_live_caret_selection_and_submit(slowly: bo] == ["X", "Y", "z", "!"]@pytest.mark.asyncioasync def test_browser_actions_fields_live_failed_field_preserves_prior_changes() -> None:async with _browser() as (client, session, page):diff --git a/tests/ai/test_browser_mouse.py b/tests/ai/test_browser_mouse.pyindex 25e4176..b554f16 100644--- a/tests/ai/test_browser_mouse.py+++ b/tests/ai/test_browser_mouse.py@@ -633,6 +633,7 @@ async def test_browser_mouse_chain_reports_failed_index_and_keeps_completed_actiassert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncioasync def test_browser_mouse_live_native_events_and_navigation() -> None:server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))@@ -708,6 +709,7 @@ async def test_browser_mouse_live_native_events_and_navigation() -> None:assert not closed.is_error@pytest.mark.asyncio@pytest.mark.parametrize("cancel_mode", ["task", "scope"])@pytest.mark.parametrize("target_type", ["xy", "ref"])@@ -774,6 +776,7 @@ async def test_browser_mouse_live_cancelled_click_releases_native_button(cancel_assert not closed.is_error@pytest.mark.asyncio@pytest.mark.parametrize("tool", ["move", "click"])async def test_browser_mouse_live_selectors_refs_and_scroll(tool: str) -> None:@@ -849,6 +852,7 @@ async def test_browser_mouse_live_selectors_refs_and_scroll(tool: str) -> None:assert not closed.is_error@pytest.mark.asyncioasync def test_browser_mouse_click_live_timeout_releases_native_button() -> None:server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))diff --git a/tests/ai/test_browser_network.py b/tests/ai/test_browser_network.pyindex 89646fa..cb2d98d 100644--- a/tests/ai/test_browser_network.py+++ b/tests/ai/test_browser_network.py@@ -873,6 +873,7 @@ async def test_network_missing_id_and_search_error_reach_client() -> None:assert result.is_error and "regular expression" in result.content[0].text@pytest.mark.asyncioasync def test_network_live_capture_actions_navigation_and_reads() -> None:server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))diff --git a/tests/ai/test_browser_press_key.py b/tests/ai/test_browser_press_key.pyindex 3863d86..0b3064c 100644--- a/tests/ai/test_browser_press_key.py+++ b/tests/ai/test_browser_press_key.py@@ -208,6 +208,7 @@ async def _browser() -> AsyncGenerator[tuple[Client, Any, Any], None]:assert not closed.is_error@pytest.mark.asyncioasync def test_browser_actions_keys_live_focus_editing_and_shortcuts() -> None:async with _browser() as (client, session, page):diff --git a/tests/ai/test_browser_screenshot.py b/tests/ai/test_browser_screenshot.pyindex 8abd249..276e5d5 100644--- a/tests/ai/test_browser_screenshot.py+++ b/tests/ai/test_browser_screenshot.py@@ -427,6 +427,7 @@ async def test_browser_screenshot_reserves_page_until_cancelled(assert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncioasync def test_browser_screenshot_live_preserves_actions_and_current_page() -> None:server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))diff --git a/tests/ai/test_browser_state.py b/tests/ai/test_browser_state.pyindex 7d85d1e..526c18a 100644--- a/tests/ai/test_browser_state.py+++ b/tests/ai/test_browser_state.py@@ -83,6 +83,7 @@ async def test_browser_state_requires_live_browser(operation: str, kind: str, mesession.load_state.assert_not_awaited()@pytest.mark.asyncioasync def test_browser_state_live_round_trip(tmp_path: Path) -> None:path, native_path = tmp_path / "state.json", tmp_path / "native.json"@@ -118,6 +119,7 @@ async def test_browser_state_live_round_trip(tmp_path: Path) -> None:assert await page.evaluate(READ_STATE) == SAVED_STATE@pytest.mark.asyncioasync def test_browser_state_live_file_errors(tmp_path: Path) -> None:invalid = tmp_path / "invalid.json"diff --git a/tests/ai/test_browser_wait.py b/tests/ai/test_browser_wait.pyindex 7e5b664..13f6bf4 100644--- a/tests/ai/test_browser_wait.py+++ b/tests/ai/test_browser_wait.py@@ -339,6 +339,7 @@ async def _browser() -> AsyncGenerator[tuple[Client, Any, Any], None]:assert not (await client.call_tool("close_session", {"session_id": "browser"})).is_error@pytest.mark.asyncioasync def test_browser_wait_live_pause_and_element_states() -> None:async with _browser() as (client, session, page):@@ -396,6 +397,7 @@ async def test_browser_wait_live_pause_and_element_states() -> None:assert session.page_pool.pages[0].state == "ready"@pytest.mark.asyncioasync def test_browser_wait_live_current_document_load_states() -> None:async with _browser() as (client, session, page):@@ -452,6 +454,7 @@ async def test_browser_wait_live_current_document_load_states() -> None:await asyncio.gather(task, return_exceptions=True)@pytest.mark.asyncioasync def test_browser_wait_live_load_visible_hidden_chain() -> None:async with _browser() as (client, session, page):diff --git a/tests/ai/test_browser_wheel.py b/tests/ai/test_browser_wheel.pyindex 2bd01d5..06654cc 100644--- a/tests/ai/test_browser_wheel.py+++ b/tests/ai/test_browser_wheel.py@@ -215,6 +215,7 @@ async def _browser() -> AsyncGenerator[tuple[Client, Any, Any], None]:assert not closed.is_error@pytest.mark.asyncioasync def test_browser_wheel_live_scrolls_under_current_pointer() -> None:async with _browser() as (client, session, page):diff --git a/tests/cli/test_browser_commands.py b/tests/cli/test_browser_commands.pynew file mode 100644index 0000000..fbcc39e--- /dev/null+++ b/tests/cli/test_browser_commands.py@@ -0,0 +1,169 @@+from inspect import Parameter, signature+from pathlib import Path+from unittest.mock import MagicMock, call, patch++import pytest+from click.testing import CliRunner++from scrapling import cli+from scrapling.core.shell import CustomShell+from scrapling.core._shell_signatures import Signatures_map+from scrapling.engines._browsers._types import StealthSession+from scrapling.fetchers import StealthyFetcher+from scrapling.parser import Selector+++ ("flags", "options"),+ [+ ([], {"block_webrtc": False, "solve_cloudflare": False, "allow_webgl": True, "hide_canvas": False}),+ (+ ["--block-webrtc", "--solve-cloudflare", "--block-webgl", "--hide-canvas"],+ {"block_webrtc": True, "solve_cloudflare": True, "allow_webgl": False, "hide_canvas": True},+ ),+ (+ ["--allow-webrtc", "--no-solve-cloudflare", "--allow-webgl", "--show-canvas"],+ {"block_webrtc": False, "solve_cloudflare": False, "allow_webgl": True, "hide_canvas": False},+ ),+ ],+)+def test_fetch_stealth_options(tmp_path: Path, flags: list[str], options: dict[str, bool]) -> None:+ with patch.object(StealthyFetcher, "fetch", return_value=Selector("<p>Content</p>")) as fetch:+ result = CliRunner().invoke(+ cli.main, ["extract", "fetch", "https://example.com", str(tmp_path / "page.txt"), *flags]+ )+ assert result.exit_code == 0, result.output+ assert options.items() <= fetch.call_args.kwargs.items()+++def test_fetch_common_options(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("SCRAPLING_EXECUTABLE_PATH", raising=False)+ output = tmp_path / "page.txt"+ with patch.object(StealthyFetcher, "fetch", return_value=Selector("<p>Content</p><b>Other</b>")) as fetch:+ result = CliRunner().invoke(+ cli.main,+ [+ "extract",+ "fetch",+ "https://example.com",+ str(output),+ "--no-headless",+ "--disable-resources",+ "--network-idle",+ "--pierce-shadow",+ "--timeout",+ "60000",+ "--wait",+ "50",+ "--css-selector",+ "p",+ "--wait-selector",+ "p",+ "--locale",+ "en-GB",+ "--real-chrome",+ "--proxy",+ "http://localhost:8080",+ "--extra-headers",+ "X-Test: value",+ "--executable-path",+ "/opt/chromium",+ "--dns-over-https",+ "--block-ads",+ ],+ )+ assert result.exit_code == 0, result.output+ fetch.assert_called_once_with(+ "https://example.com",+ headless=False,+ disable_resources=True,+ network_idle=True,+ pierce_shadow=True,+ timeout=60000,+ wait=50,+ wait_selector="p",+ locale="en-GB",+ real_chrome=True,+ proxy="http://localhost:8080",+ extra_headers={"X-Test": "value"},+ executable_path="/opt/chromium",+ dns_over_https=True,+ block_ads=True,+ block_webrtc=False,+ solve_cloudflare=False,+ allow_webgl=True,+ hide_canvas=False,+ )+ assert output.read_text() == "Content"+++def test_browser_command_help_and_removed_command() -> None:+ runner = CliRunner()+ result = runner.invoke(cli.main, ["extract", "fetch", "--help"])+ assert result.exit_code == 0+ assert "StealthyFetcher" in result.output+ for option in ("--block-webrtc", "--solve-cloudflare", "--allow-webgl", "--hide-canvas"):+ assert option in result.output+ result = runner.invoke(cli.main, ["extract", "stealthy-fetch", "--help"])+ assert result.exit_code == 2+ assert "No such command 'stealthy-fetch'" in result.output+ assert not hasattr(cli, "stealthy_fetch")++[email protected](("installed", "force"), [(False, False), (True, False), (True, True), (False, True)])+def test_install_playwright_keeps_marker(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, installed: bool, force: bool+) -> None:+ marker = tmp_path / ".scrapling_dependencies_installed"+ if installed:+ marker.touch()+ monkeypatch.setattr(cli, "__PACKAGE_DIR__", tmp_path)+ with patch("scrapling.cli.check_output") as execute, patch("tld.utils.update_tld_names") as update:+ result = CliRunner().invoke(cli.main, ["install", *(["--force"] if force else [])])+ assert result.exit_code == 0, result.output+ assert marker.exists()+ if force or not installed:+ assert execute.call_args_list == [+ call([cli.python_executable, "-m", "playwright", "install", "chromium"], shell=False),+ call([cli.python_executable, "-m", "playwright", "install-deps", "chromium"], shell=False),+ ]+ update.assert_called_once_with(fail_silently=True)+ assert "Installing browsers" in result.output+ else:+ execute.assert_not_called()+ update.assert_not_called()+ assert result.output.strip() == "The dependencies are already installed"+++def test_shell_fetch_namespace_and_completion() -> None:+ shell = CustomShell(code="")+ namespace = shell.get_namespace()+ assert namespace["StealthyFetcher"] is StealthyFetcher+ assert {"StealthySession", "AsyncStealthySession", "fetch"} <= namespace.keys()+ assert (+ not {"DynamicFetcher", "DynamicSession", "AsyncDynamicSession", "PlayWrightFetcher", "stealthy_fetch"}+ & namespace.keys()+ )+ parameters = signature(namespace["fetch"]).parameters+ assert set(parameters) == {"url", *StealthSession.__annotations__}+ assert all(parameters[name].kind is Parameter.KEYWORD_ONLY for name in StealthSession.__annotations__)+ assert Signatures_map["fetch"] == StealthSession.__annotations__+ assert "stealthy_fetch" not in Signatures_map+ assert "Shortcut for `StealthyFetcher.fetch`" in shell.banner()+ assert "Dynamic" not in shell.banner()+ assert "stealthy_fetch" not in shell.banner()+++def test_shell_fetch_updates_page_history() -> None:+ pages = [Selector(f"<p>{index}</p>") for index in range(7)]+ shell = CustomShell(code="")+ shell.shell = MagicMock(user_ns={})+ with patch.object(StealthyFetcher, "fetch", side_effect=pages) as fetch:+ namespace = shell.get_namespace()+ for page in pages:+ assert namespace["fetch"]("https://example.com", solve_cloudflare=True) is page+ assert fetch.call_count == len(pages)+ fetch.assert_called_with("https://example.com", solve_cloudflare=True)+ assert shell.page is pages[-1]+ assert shell.pages == pages[-5:]+ assert shell.shell.user_ns == {"page": pages[-1], "response": pages[-1], "pages": pages[-5:]}diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.pyindex dde4db6..27d164c 100644--- a/tests/cli/test_cli.py+++ b/tests/cli/test_cli.py@@ -5,7 +5,7 @@ import pytest_httpbinfrom scrapling.parser import Selectorfrom scrapling import __version__-from scrapling.cli import main, shell, mcp, get, post, put, delete, fetch, stealthy_fetch+from scrapling.cli import main, shell, mcp, get, post, put, delete, fetch@pytest_httpbin.use_class_based_httpbin@@ -215,33 +215,19 @@ class TestCLI:"""Test extract fetch command"""output_file = tmp_path / "output.txt"- with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:- mock_response = configure_selector_mock()- mock_fetch.return_value = mock_response-- result = runner.invoke(fetch, [html_url, str(output_file), "--headless", "--timeout", "60000"])- assert result.exit_code == 0-- def test_extract_stealthy_fetch_command(self, runner, tmp_path, html_url):- """Test extract fetch command"""- output_file = tmp_path / "output.md"-with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:mock_response = configure_selector_mock()mock_fetch.return_value = mock_response- result = runner.invoke(- stealthy_fetch,- [html_url, str(output_file), "--headless", "--css-selector", "body", "--timeout", "60000"],- )+ result = runner.invoke(fetch, [html_url, str(output_file), "--headless", "--timeout", "60000"])assert result.exit_code == 0def test_extract_fetch_with_executable_path(self, runner, tmp_path, html_url, monkeypatch):- """Test that --executable-path is passed through to DynamicFetcher and wins over the environment variable"""+ """Test that --executable-path is passed through to StealthyFetcher and wins over the environment variable"""output_file = tmp_path / "output.html"monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/env-chromium")- with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:+ with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:mock_fetch.return_value = configure_selector_mock()result = runner.invoke(fetch, [html_url, str(output_file), "--executable-path", "/opt/custom-chromium"])@@ -253,7 +239,7 @@ class TestCLI:output_file = tmp_path / "output.html"monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/env-chromium")- with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:+ with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:mock_fetch.return_value = configure_selector_mock()result = runner.invoke(fetch, [html_url, str(output_file)])@@ -265,35 +251,13 @@ class TestCLI:output_file = tmp_path / "output.html"monkeypatch.delenv("SCRAPLING_EXECUTABLE_PATH", raising=False)- with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:+ with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:mock_fetch.return_value = configure_selector_mock()result = runner.invoke(fetch, [html_url, str(output_file)])assert result.exit_code == 0assert "executable_path" not in mock_fetch.call_args.kwargs- def test_extract_stealthy_fetch_with_executable_path(self, runner, tmp_path, html_url, monkeypatch):- """Test that --executable-path and the environment fallback work for stealthy_fetch too"""- output_file = tmp_path / "output.html"- monkeypatch.delenv("SCRAPLING_EXECUTABLE_PATH", raising=False)-- with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:- mock_fetch.return_value = configure_selector_mock()-- result = runner.invoke(- stealthy_fetch, [html_url, str(output_file), "--executable-path", "/opt/custom-chromium"]- )- assert result.exit_code == 0- assert mock_fetch.call_args.kwargs["executable_path"] == "/opt/custom-chromium"-- monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/env-chromium")- with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:- mock_fetch.return_value = configure_selector_mock()-- result = runner.invoke(stealthy_fetch, [html_url, str(output_file)])- assert result.exit_code == 0- assert mock_fetch.call_args.kwargs["executable_path"] == "/opt/env-chromium"-def test_invalid_arguments(self, runner, html_url):"""Test invalid arguments handling"""# Missing required argumentsdiff --git a/tests/cli/test_shadow_cli.py b/tests/cli/test_shadow_cli.pyindex 097a5e2..d7e8121 100644--- a/tests/cli/test_shadow_cli.py+++ b/tests/cli/test_shadow_cli.py@@ -8,24 +8,20 @@ from scrapling.cli import mainfrom scrapling.parser import Selector[email protected](("command", "fetcher"), [("fetch", "DynamicFetcher"), ("stealthy-fetch", "StealthyFetcher")])@pytest.mark.parametrize(("flags", "enabled"), [([], False), (["--pierce-shadow"], True), (["--no-pierce-shadow"], False)])-def test_shadow_option_reaches_fetcher(- tmp_path: Path, command: str, fetcher: str, flags: list[str], enabled: bool-) -> None:+def test_shadow_option_reaches_fetcher(tmp_path: Path, flags: list[str], enabled: bool) -> None:output = tmp_path / "page.txt"- with patch(f"scrapling.fetchers.{fetcher}.fetch", return_value=Selector("<p>Content</p>")) as fetch:- result = CliRunner().invoke(main, ["extract", command, "https://example.com", str(output), *flags])+ with patch("scrapling.fetchers.StealthyFetcher.fetch", return_value=Selector("<p>Content</p>")) as fetch:+ result = CliRunner().invoke(main, ["extract", "fetch", "https://example.com", str(output), *flags])assert result.exit_code == 0, result.outputassert fetch.call_args.kwargs["pierce_shadow"] is enabledassert output.read_text() == "Content"[email protected]("command", ["fetch", "stealthy-fetch"])-def test_shadow_option_in_browser_help(command: str) -> None:- result = CliRunner().invoke(main, ["extract", command, "--help"])+def test_shadow_option_in_browser_help() -> None:+ result = CliRunner().invoke(main, ["extract", "fetch", "--help"])assert result.exit_code == 0assert "--pierce-shadow / --no-pierce-shadow" in result.outputdiff --git a/tests/cli/test_shell_functionality.py b/tests/cli/test_shell_functionality.pyindex d23fa22..995e657 100644--- a/tests/cli/test_shell_functionality.py+++ b/tests/cli/test_shell_functionality.py@@ -2,6 +2,7 @@ import pytestfrom unittest.mock import patch, MagicMockfrom scrapling.parser import Selector+from scrapling.core._types import cast, extraction_typesfrom scrapling.core.shell import CustomShell, CurlParser, Convertor@@ -15,11 +16,11 @@ class TestCurlParser:def test_basic_curl_parse(self, parser):"""Test parsing basic curl commands"""# Simple GET- curl_cmd = 'curl https://example.com'+ curl_cmd = "curl https://example.com"request = parser.parse(curl_cmd)- assert request.url == 'https://example.com'- assert request.method == 'get'+ assert request.url == "https://example.com"+ assert request.method == "get"assert request.data is Nonedef test_curl_with_headers(self, parser):@@ -30,8 +31,8 @@ class TestCurlParser:request = parser.parse(curl_cmd)- assert request.headers['User-Agent'] == 'Mozilla/5.0'- assert request.headers['Accept'] == 'application/json'+ assert request.headers["User-Agent"] == "Mozilla/5.0"+ assert request.headers["Accept"] == "application/json"def test_curl_with_data(self, parser):"""Test parsing curl with data"""@@ -39,8 +40,8 @@ class TestCurlParser:curl_cmd = 'curl https://example.com -X POST -d "key=value&foo=bar"'request = parser.parse(curl_cmd)- assert request.method == 'post'- assert request.data == 'key=value&foo=bar'+ assert request.method == "post"+ assert request.data == "key=value&foo=bar"# JSON datacurl_cmd = """curl https://example.com -X POST --data-raw '{"key": "value"}'"""@@ -56,24 +57,24 @@ class TestCurlParser:request = parser.parse(curl_cmd)- assert request.cookies['session'] == 'abc123'- assert request.cookies['user'] == 'john'- assert request.cookies['extra'] == 'cookie'+ assert request.cookies["session"] == "abc123"+ assert request.cookies["user"] == "john"+ assert request.cookies["extra"] == "cookie"def test_curl_with_proxy(self, parser):"""Test parsing curl with proxy"""- curl_cmd = 'curl https://example.com -x http://proxy:8080 -U user:pass'+ curl_cmd = "curl https://example.com -x http://proxy:8080 -U user:pass"request = parser.parse(curl_cmd)- assert 'http://user:pass@proxy:8080' in request.proxy['http']+ assert "http://user:pass@proxy:8080" in request.proxy["http"]def test_curl2fetcher(self, parser):"""Test converting curl to fetcher request"""- with patch('scrapling.fetchers.Fetcher.get') as mock_get:+ with patch("scrapling.fetchers.Fetcher.get") as mock_get:mock_response = MagicMock()mock_get.return_value = mock_response- curl_cmd = 'curl https://example.com'+ curl_cmd = "curl https://example.com"_ = parser.convert2fetcher(curl_cmd)mock_get.assert_called_once()@@ -82,7 +83,7 @@ class TestCurlParser:"""Test handling invalid curl commands"""# Invalid formatwith pytest.raises(AttributeError):- parser.parse('not a curl command')+ parser.parse("not a curl command")class TestConvertor:@@ -129,11 +130,7 @@ class TestConvertor:def test_extract_with_selector(self, sample_html):"""Test extracting with CSS selector"""page = Selector(sample_html)- content = list(Convertor._extract_content(- page,- "text",- css_selector=".content"- ))+ content = list(Convertor._extract_content(page, "text", css_selector=".content"))assert len(content) > 0@@ -162,7 +159,7 @@ class TestConvertor:# Invalid extraction typewith pytest.raises(ValueError):- list(Convertor._extract_content(page, "invalid"))+ list(Convertor._extract_content(page, cast(extraction_types, "invalid")))# Invalid filenamewith pytest.raises(ValueError):@@ -190,9 +187,9 @@ class TestCustomShell:namespace = shell.get_namespace()# Check all expected functions/classes are available- assert 'get' in namespace- assert 'post' in namespace- assert 'Fetcher' in namespace- assert 'DynamicFetcher' in namespace- assert 'view' in namespace- assert 'uncurl' in namespace+ assert "get" in namespace+ assert "post" in namespace+ assert "Fetcher" in namespace+ assert "StealthyFetcher" in namespace+ assert "view" in namespace+ assert "uncurl" in namespacediff --git a/tests/fetchers/async/test_dynamic.py b/tests/fetchers/async/test_dynamic.pydeleted file mode 100644index 3fe38cc..0000000--- a/tests/fetchers/async/test_dynamic.py+++ /dev/null@@ -1,90 +0,0 @@-import pytest-import pytest_httpbin--from scrapling import DynamicFetcher--DynamicFetcher.adaptive = True---@pytest_httpbin.use_class_based_httpbin-class TestDynamicFetcherAsync:- @pytest.fixture- def fetcher(self):- return DynamicFetcher-- @pytest.fixture- def urls(self, httpbin):- return {- "status_200": f"{httpbin.url}/status/200",- "status_404": f"{httpbin.url}/status/404",- "status_501": f"{httpbin.url}/status/501",- "basic_url": f"{httpbin.url}/get",- "html_url": f"{httpbin.url}/html",- "delayed_url": f"{httpbin.url}/delay/10",- "cookies_url": f"{httpbin.url}/cookies/set/test/value",- }-- @pytest.mark.asyncio- async def test_basic_fetch(self, fetcher, urls):- """Test doing a basic fetch request with multiple statuses"""- response = await fetcher.async_fetch(urls["status_200"])- assert response.status == 200-- @pytest.mark.asyncio- async def test_cookies_loading(self, fetcher, urls):- """Test if cookies are set after the request"""- response = await fetcher.async_fetch(urls["cookies_url"])- cookies = {response.cookies[0]['name']: response.cookies[0]['value']}- assert cookies == {"test": "value"}-- @pytest.mark.asyncio- async def test_automation(self, fetcher, urls):- """Test if automation breaks the code or not"""-- async def scroll_page(page):- await page.mouse.wheel(10, 0)- await page.mouse.move(100, 400)- await page.mouse.up()- return page-- response = await fetcher.async_fetch(urls["html_url"], page_action=scroll_page)- assert response.status == 200-- @pytest.mark.parametrize(- "kwargs",- [- {"real_chrome": True, "disable_resources": True},- {"wait_selector": "h1", "wait_selector_state": "attached"},- {"wait_selector": "h1", "wait_selector_state": "visible"},- {- "google_search": True,- "real_chrome": True,- "wait": 10,- "locale": "en-US",- "extra_headers": {"ayo": ""},- "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0",- "cookies": [{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],- "network_idle": True,- "selector_config": {"keep_comments": False, "keep_cdata": False},- },- ],- )- @pytest.mark.asyncio- async def test_properties(self, fetcher, urls, kwargs):- """Test if different arguments break the code or not"""- response = await fetcher.async_fetch(urls["html_url"], **kwargs)- assert response.status == 200-- @pytest.mark.asyncio- async def test_cdp_url_invalid(self, fetcher, urls):- """Test if invalid CDP URLs raise appropriate exceptions"""- with pytest.raises(TypeError):- await fetcher.async_fetch(urls["html_url"], cdp_url="blahblah")-- with pytest.raises(TypeError):- await fetcher.async_fetch(- urls["html_url"], cdp_url="blahblah"- )-- with pytest.raises(Exception):- await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")diff --git a/tests/fetchers/async/test_dynamic_session.py b/tests/fetchers/async/test_dynamic_session.pydeleted file mode 100644index 9c8671a..0000000--- a/tests/fetchers/async/test_dynamic_session.py+++ /dev/null@@ -1,139 +0,0 @@-import pytest-import asyncio--import pytest_httpbin--from scrapling.fetchers import AsyncDynamicSession---@pytest_httpbin.use_class_based_httpbin-class TestAsyncDynamicSession:- """Test AsyncDynamicSession"""-- # The `AsyncDynamicSession` is inheriting from `DynamicSession` class so no need to repeat all the tests- @pytest.fixture- def urls(self, httpbin):- return {- "basic": f"{httpbin.url}/get",- "html": f"{httpbin.url}/html",- }-- async def test_concurrent_async_requests(self, urls):- """Test concurrent requests with async session"""- async with AsyncDynamicSession(max_pages=3) as session:- # Launch multiple concurrent requests- tasks = [- session.fetch(urls["basic"]),- session.fetch(urls["html"]),- session.fetch(urls["basic"])- ]-- assert session.max_pages == 3- assert session.page_pool.max_pages == 3- assert session.context is not None-- responses = await asyncio.gather(*tasks)-- # All should succeed- assert all(r.status == 200 for r in responses)-- # Check pool stats- stats = session.get_pool_stats()- assert stats["total_pages"] <= 3-- # After exit, should be closed- assert session._is_alive is False-- # Should raise RuntimeError when used after closing- with pytest.raises(RuntimeError):- await session.fetch(urls["basic"])-- async def test_page_pool_management(self, urls):- """Test page pool creation and reuse"""- async with AsyncDynamicSession() as session:- # The first request creates a page, and it stays open afterwards- response = await session.fetch(urls["basic"])- assert response.status == 200- assert session.page_pool.pages_count == 1- page = session.page_pool.pages[0].page- assert session.page_pool.pages[0].state == "ready"-- # The second request reuses the same tab- response = await session.fetch(urls["html"])- assert response.status == 200- assert session.page_pool.pages_count == 1- assert session.page_pool.pages[0].page is page-- # Check pool stats- stats = session.get_pool_stats()- assert stats["total_pages"] == 1- assert stats["busy_pages"] == 0- assert stats["max_pages"] == 1-- async def test_close_pages(self, urls):- """Closing the tabs empties the pool, and the next request opens a fresh one"""- async with AsyncDynamicSession() as session:- await session.fetch(urls["basic"])- page = session.page_pool.pages[0].page-- await session.close_pages()- assert session.page_pool.pages_count == 0- assert page.is_closed()-- response = await session.fetch(urls["html"])- assert response.status == 200- assert session.page_pool.pages_count == 1- assert session.page_pool.pages[0].page is not page-- async def test_per_request_headers_do_not_leak_into_the_next_request(self, httpbin):- """A reused tab gets its settings reset, so headers from the previous request are gone"""- async with AsyncDynamicSession() as session:- response = await session.fetch(f"{httpbin.url}/headers", extra_headers={"X-Scrapling-First": "1"})- assert "X-Scrapling-First" in response.get_all_text()-- response = await session.fetch(f"{httpbin.url}/headers", extra_headers={"X-Scrapling-Second": "2"})- assert "X-Scrapling-Second" in response.get_all_text()- assert "X-Scrapling-First" not in response.get_all_text(), "the new headers must replace the old ones"-- response = await session.fetch(f"{httpbin.url}/headers")- assert "X-Scrapling-Second" not in response.get_all_text()- assert "X-Scrapling-First" not in response.get_all_text()-- async def test_resource_blocking_does_not_leak_into_the_next_request(self, httpbin):- """Routes registered for one request are removed before the tab is reused"""- js = """async () => {- const img = document.createElement('img');- const done = new Promise(r => { img.onload = () => r('loaded'); img.onerror = () => r('blocked'); });- img.src = '/image/png?' + Math.random();- document.body.appendChild(img);- return await done;- }"""- results = []-- async def probe(page):- results.append(await page.evaluate(js))-- async with AsyncDynamicSession() as session:- await session.fetch(f"{httpbin.url}/html", disable_resources=True, page_action=probe)- await session.fetch(f"{httpbin.url}/html", page_action=probe)- assert results == ["blocked", "loaded"]- assert session.page_pool.pages_count == 1-- async def test_dynamic_session_with_options(self, urls):- """Test AsyncDynamicSession with various options"""- async with AsyncDynamicSession(- headless=False,- disable_resources=True,- extra_headers={"X-Test": "value"}- ) as session:- response = await session.fetch(urls["html"])- assert response.status == 200-- async def test_error_handling_in_fetch(self, urls):- """Test error handling during fetch"""- async with AsyncDynamicSession() as session:- # Test with invalid URL- with pytest.raises(Exception):- await session.fetch("invalid://url")- assert session.page_pool.pages_count == 0, "errored tabs are closed and evicted"diff --git a/tests/fetchers/async/test_stealth.py b/tests/fetchers/async/test_stealth.pyindex bba39d8..3e0abfc 100644--- a/tests/fetchers/async/test_stealth.py+++ b/tests/fetchers/async/test_stealth.py@@ -6,6 +6,7 @@ from scrapling import StealthyFetcherStealthyFetcher.adaptive = True@pytest_httpbin.use_class_based_httpbin@pytest.mark.asyncioclass TestStealthyFetcher:@@ -23,7 +24,7 @@ class TestStealthyFetcher:"basic_url": f"{url}/get","html_url": f"{url}/html","delayed_url": f"{url}/delay/10", # 10 Seconds delay response- "cookies_url": f"{url}/cookies/set/test/value"+ "cookies_url": f"{url}/cookies/set/test/value",}async def test_basic_fetch(self, fetcher, urls):@@ -35,10 +36,11 @@ class TestStealthyFetcher:async def test_cookies_loading(self, fetcher, urls):"""Test if cookies are set after the request"""response = await fetcher.async_fetch(urls["cookies_url"])- cookies = {response.cookies[0]['name']: response.cookies[0]['value']}+ cookies = {response.cookies[0]["name"]: response.cookies[0]["value"]}assert cookies == {"test": "value"}- async def test_automation(self, fetcher, urls):+ @pytest.mark.parametrize("humanize", [False, True])+ async def test_automation(self, fetcher, urls, humanize):"""Test if automation breaks the code or not"""async def scroll_page(page):@@ -47,13 +49,12 @@ class TestStealthyFetcher:await page.mouse.up()return page- assert (- await fetcher.async_fetch(urls["html_url"], page_action=scroll_page, humanize=True)- ).status == 200+ assert (await fetcher.async_fetch(urls["html_url"], page_action=scroll_page, humanize=humanize)).status == 200@pytest.mark.parametrize("kwargs",[+ {"real_chrome": True, "disable_resources": True},{"block_webrtc": True, "allow_webgl": True},{"block_webrtc": False, "allow_webgl": True},{"block_webrtc": True, "allow_webgl": False, "disable_resources": True},@@ -64,6 +65,9 @@ class TestStealthyFetcher:"wait": 10,"cookies": [{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],"google_search": True,+ "real_chrome": True,+ "locale": "en-US",+ "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0","extra_headers": {"ayo": ""},"selector_config": {"keep_comments": False, "keep_cdata": False},"additional_args": {},@@ -72,8 +76,11 @@ class TestStealthyFetcher:)async def test_properties(self, fetcher, urls, kwargs):"""Test if different arguments break the code or not"""- response = await fetcher.async_fetch(- urls["html_url"],- **kwargs- )+ response = await fetcher.async_fetch(urls["html_url"], **kwargs)assert response.status == 200++ async def test_cdp_url_invalid(self, fetcher, urls):+ with pytest.raises(TypeError):+ await fetcher.async_fetch(urls["html_url"], cdp_url="blahblah")+ with pytest.raises(Exception):+ await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")diff --git a/tests/fetchers/async/test_stealth_session.py b/tests/fetchers/async/test_stealth_session.pyindex 15ea981..9035163 100644--- a/tests/fetchers/async/test_stealth_session.py+++ b/tests/fetchers/async/test_stealth_session.py@@ -1,4 +1,3 @@-import pytestimport asyncio@@ -7,6 +6,7 @@ import pytest_httpbinfrom scrapling.fetchers import AsyncStealthySession@pytest_httpbin.use_class_based_httpbin@pytest.mark.asyncioclass TestAsyncStealthySession:@@ -24,11 +24,7 @@ class TestAsyncStealthySession:"""Test concurrent requests with async session"""async with AsyncStealthySession(max_pages=3) as session:# Launch multiple concurrent requests- tasks = [- session.fetch(urls["basic"]),- session.fetch(urls["html"]),- session.fetch(urls["basic"])- ]+ tasks = [session.fetch(urls["basic"]), session.fetch(urls["html"]), session.fetch(urls["basic"])]assert session.max_pages == 3assert session.page_pool.max_pages == 3@@ -121,13 +117,16 @@ class TestAsyncStealthySession:assert results == ["blocked", "loaded"]assert session.page_pool.pages_count == 1- async def test_stealthy_session_with_options(self, urls):+ @pytest.mark.parametrize(+ "options",+ [+ {"max_pages": 1, "block_webrtc": True, "allow_webgl": True},+ {"headless": False, "disable_resources": True, "extra_headers": {"X-Test": "value"}},+ ],+ )+ async def test_stealthy_session_with_options(self, urls, options):"""Test AsyncStealthySession with various options"""- async with AsyncStealthySession(- max_pages=1,- block_webrtc=True,- allow_webgl=True- ) as session:+ async with AsyncStealthySession(**options) as session:response = await session.fetch(urls["html"])assert response.status == 200diff --git a/tests/fetchers/sync/test_dynamic.py b/tests/fetchers/sync/test_stealth.pysimilarity index 80%rename from tests/fetchers/sync/test_dynamic.pyrename to tests/fetchers/sync/test_stealth.pyindex cac8a73..8fcaf26 100644--- a/tests/fetchers/sync/test_dynamic.py+++ b/tests/fetchers/sync/test_stealth.py@@ -1,17 +1,18 @@import pytestimport pytest_httpbin-from scrapling import DynamicFetcher+from scrapling import StealthyFetcher-DynamicFetcher.adaptive = True+StealthyFetcher.adaptive = True@pytest_httpbin.use_class_based_httpbin-class TestDynamicFetcher:+class TestStealthyFetcher:@pytest.fixture(scope="class")def fetcher(self):"""Fixture to create a StealthyFetcher instance for the entire test class"""- return DynamicFetcher+ return StealthyFetcher@pytest.fixture(autouse=True)def setup_urls(self, httpbin):@@ -21,20 +22,17 @@ class TestDynamicFetcher:self.status_501 = f"{httpbin.url}/status/501"self.basic_url = f"{httpbin.url}/get"self.html_url = f"{httpbin.url}/html"- self.delayed_url = f"{httpbin.url}/delay/10" # 10 Seconds delay response+ self.delayed_url = f"{httpbin.url}/delay/10"self.cookies_url = f"{httpbin.url}/cookies/set/test/value"def test_basic_fetch(self, fetcher):"""Test doing a basic fetch request with multiple statuses"""assert fetcher.fetch(self.status_200).status == 200- # There's a bug with playwright makes it crashes if a URL returns status code 4xx/5xx without body, let's disable this till they reply to my issue report- # assert fetcher.fetch(self.status_404).status == 404- # assert fetcher.fetch(self.status_501).status == 501def test_cookies_loading(self, fetcher):"""Test if cookies are set after the request"""response = fetcher.fetch(self.cookies_url)- cookies = {response.cookies[0]['name']: response.cookies[0]['value']}+ cookies = {response.cookies[0]["name"]: response.cookies[0]["value"]}assert cookies == {"test": "value"}def test_automation(self, fetcher):@@ -78,8 +76,5 @@ class TestDynamicFetcher:with pytest.raises(TypeError):fetcher.fetch(self.html_url, cdp_url="blahblah")- with pytest.raises(TypeError):- fetcher.fetch(self.html_url, cdp_url="blahblah")-with pytest.raises(Exception):fetcher.fetch(self.html_url, cdp_url="ws://blahblah")diff --git a/tests/fetchers/sync/test_stealth_session.py b/tests/fetchers/sync/test_stealth_session.pyindex 5285b35..d60e5bd 100644--- a/tests/fetchers/sync/test_stealth_session.py+++ b/tests/fetchers/sync/test_stealth_session.py@@ -3,7 +3,6 @@ import pytestimport pytest_httpbinfrom scrapling.engines.constants import DEFAULT_ARGS, STEALTH_ARGS-from scrapling.engines._browsers._controllers import DynamicSessionfrom scrapling.engines._browsers._stealth import StealthySession, __CF_PATTERN__@@ -19,7 +18,7 @@ class TestStealthConstants:test_urls = ["https://challenges.cloudflare.com/cdn-cgi/challenge-platform/h/123456","https://challenges.cloudflare.com/cdn-cgi/challenge-platform/orchestrate/jsch/v1",- "http://challenges.cloudflare.com/cdn-cgi/challenge-platform/scripts/abc"+ "http://challenges.cloudflare.com/cdn-cgi/challenge-platform/scripts/abc",]for url in test_urls:@@ -29,7 +28,7 @@ class TestStealthConstants:non_matching_urls = ["https://example.com/challenge","https://cloudflare.com/something",- "https://challenges.cloudflare.com/other-path"+ "https://challenges.cloudflare.com/other-path",]for url in non_matching_urls:@@ -69,20 +68,12 @@ class TestLaunchFlags:):assert flag in with_extra, f"{flag} lost to extra_flags"- def test_dynamic_session_keeps_its_defaults(self):- """DynamicSession passes no bundle, so it never lost a flag - a control that- held before this fix and pins the path the fix also touches."""- session = DynamicSession(headless=True, extra_flags=[self.USER_FLAG])- args = list(session._browser_options["args"])-- assert self.USER_FLAG in args- assert set(DEFAULT_ARGS) <= set(args), "a default flag was dropped"-@pytest_httpbin.use_class_based_httpbinclass TestStealthySession:-"""All the code is tested in the async version tests, so no need to repeat it here. The async class inherits from this one."""+@pytest.fixture(autouse=True)def setup_urls(self, httpbin):"""Fixture to set up URLs for testing"""@@ -105,7 +96,6 @@ class TestStealthySession:timeout=60000,cookies=[{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],) as session:-assert session.max_pages == 1assert session._config.headless is Trueassert session._config.disable_resources is True@@ -115,7 +105,7 @@ class TestStealthySession:assert session.context is not None# Test Cloudflare detection- for cloudflare_type in ('managed', 'interactive', 'non-interactive'):+ for cloudflare_type in ("managed", "interactive", "non-interactive"):page_content = f"""<html><script>diff --git a/tests/fetchers/test_cloudflare_solver.py b/tests/fetchers/test_cloudflare_solver.pyindex e9915c7..7184c4a 100644--- a/tests/fetchers/test_cloudflare_solver.py+++ b/tests/fetchers/test_cloudflare_solver.py@@ -2,7 +2,6 @@ from unittest.mock import AsyncMock, MagicMockimport pytestfrom patchright._impl._errors import Error as PatchrightError-from playwright._impl._errors import Error as PlaywrightErrorfrom scrapling.engines.toolbelt.convertor import ResponseFactoryfrom scrapling.engines._browsers._stealth import StealthySession, AsyncStealthySession, __CF_MAX_SOLVE_ATTEMPTS__@@ -109,20 +108,18 @@ class TestAsyncSolver:class TestPageContentRetries:- @pytest.mark.parametrize("error_class", [PatchrightError, PlaywrightError])- def test_sync_content_errors_are_retried(self, error_class):- """Both patchright and playwright errors must trigger the retry workaround, not crash"""+ def test_sync_content_errors_are_retried(self):+ """Patchright errors must trigger the retry workaround"""page = MagicMock()- page.content.side_effect = [error_class("Page.content: page is navigating"), CLEAN_PAGE]+ page.content.side_effect = [PatchrightError("Page.content: page is navigating"), CLEAN_PAGE]assert ResponseFactory._get_page_content(page) == CLEAN_PAGEassert page.wait_for_timeout.call_count == 1@pytest.mark.asyncio- @pytest.mark.parametrize("error_class", [PatchrightError, PlaywrightError])- async def test_async_content_errors_are_retried(self, error_class):+ async def test_async_content_errors_are_retried(self):page = MagicMock()- page.content = AsyncMock(side_effect=[error_class("Page.content: page is navigating"), CLEAN_PAGE])+ page.content = AsyncMock(side_effect=[PatchrightError("Page.content: page is navigating"), CLEAN_PAGE])page.wait_for_timeout = AsyncMock()assert await ResponseFactory._get_async_page_content(page) == CLEAN_PAGE@@ -154,11 +151,3 @@ class TestLocaleLaunchFlags:"""Remote browsers can't take launch flags, so the context option is the best effort left"""session = StealthySession(locale="fr-FR", cdp_url="ws://127.0.0.1:9222/devtools/browser/x")assert session._context_options.get("locale") == "fr-FR"-- def test_dynamic_session_gets_the_same_flags(self):- from scrapling.engines._browsers._controllers import DynamicSession-- session = DynamicSession(locale="en-GB")- assert "--lang=en-GB" in session._browser_options["args"]- assert "--accept-lang=en-GB,en" in session._browser_options["args"]- assert "locale" not in session._context_optionsdiff --git a/tests/fetchers/test_network_browser.py b/tests/fetchers/test_network_browser.pyindex 90ba4df..b42ed82 100644--- a/tests/fetchers/test_network_browser.py+++ b/tests/fetchers/test_network_browser.py@@ -15,12 +15,10 @@ from urllib.parse import urlsplitimport pytestfrom patchright.async_api import Response as AsyncPatchrightResponsefrom patchright.sync_api import Response as PatchrightResponse-from playwright.async_api import Response as AsyncPlaywrightResponse-from playwright.sync_api import Response as PlaywrightResponsefrom scrapling.core._types import Any, Generatorfrom scrapling.core.ai._network_formatting import _request_details-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession+from scrapling.fetchers import AsyncStealthySession, StealthySessionfrom scrapling.engines.toolbelt.custom import Responsefrom scrapling.engines.toolbelt.proxy_rotation import ProxyRotator@@ -242,7 +240,7 @@ def _options(**kwargs: Any) -> dict[str, Any]:}[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_network_disabled_by_default(session_type: Any) -> None:session = session_type()context = Mock()@@ -255,7 +253,7 @@ def test_sync_network_disabled_by_default(session_type: Any) -> None:@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_network_disabled_by_default(session_type: Any) -> None:session = session_type()context = Mock()@@ -338,7 +336,8 @@ def _check_later(network: Any, after_id: int) -> tuple[Any, Any, Any]:return post[0], status[0], network.search(url_pattern=r"/api/large$")[0][email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:with session_type(**_options(@@ -407,8 +406,9 @@ def test_sync_network_browser_history(session_type: Any, network_url: str, releaassert post.headers["x-recorded"] == "response"@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:async def initial_page(page: Any) -> None:await page.locator("body[data-initial=ready]").wait_for(state="attached")@@ -493,7 +493,8 @@ def _check_eviction(network: Any) -> int:return last_id[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_network_browser_eviction_and_clear(session_type: Any, network_url: str) -> None:with session_type(**_options(max_recorded_requests=2)) as session:session.fetch(network_url + "/")@@ -513,8 +514,9 @@ def test_sync_network_browser_eviction_and_clear(session_type: Any, network_url:assert records[0].meta["request_body"] == records[0].body == b"fresh"@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_network_browser_eviction_and_clear(session_type: Any, network_url: str) -> None:async with session_type(**_options(max_recorded_requests=2)) as session:await session.fetch(network_url + "/")@@ -537,7 +539,8 @@ async def test_async_network_browser_eviction_and_clear(session_type: Any, netwoPROXY_HTML = '<html><body><script>fetch("/api").then(response => response.json()).then(value => document.body.dataset.api = JSON.stringify(value));</script></body></html>'[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_network_browser_temporary_proxy_context(session_type: Any) -> None:def setup(page: Any) -> None:page.route(@@ -564,8 +567,9 @@ def test_sync_network_browser_temporary_proxy_context(session_type: Any) -> Noneassert record.json() == {"temporary": True}@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_network_browser_temporary_proxy_context(session_type: Any) -> None:async def setup(page: Any) -> None:await page.route(@@ -602,7 +606,8 @@ def _check_saved_override(network: Any, network_url: str) -> None:assert not network.search(url_pattern=r"/api/original$")[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_network_browser_keeps_route_overrides_after_close(session_type: Any, network_url: str, monkeypatch: Any) -> None:@@ -628,8 +633,9 @@ def test_sync_network_browser_keeps_route_overrides_after_close(assert f"<GET {network_url}/>" in logger.info.call_args.args[0]@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_network_browser_keeps_route_overrides_after_close(session_type: Any, network_url: str, monkeypatch: Any) -> None:@@ -655,7 +661,8 @@ async def test_async_network_browser_keeps_route_overrides_after_close(assert f"<GET {network_url}/>" in logger.info.call_args.args[0][email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])@pytest.mark.parametrize("target", ["page", "context", "session"])def test_sync_network_close_during_unfinished_response(session_type: Any, target: str, network_url: str, network_gates: dict[str, _Gate]@@ -689,8 +696,9 @@ def test_sync_network_close_during_unfinished_response(assert saved.body == BLANK@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])@pytest.mark.parametrize("target", ["page", "context", "session"])async def test_async_network_close_during_unfinished_response(session_type: Any, target: str, network_url: str, network_gates: dict[str, _Gate]@@ -737,7 +745,8 @@ def _check_burst(network: Any, cursor: int, path: str, expected_paths: list[str]return newer[0].meta["network_id"][email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_network_burst_completion_order_and_pagination(session_type: Any, network_url: str, network_gates: dict[str, _Gate]) -> None:@@ -775,8 +784,9 @@ def test_sync_network_burst_completion_order_and_pagination(assert all(record.json() == {"ok": True} for record in records)@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_network_burst_completion_order_and_pagination(session_type: Any, network_url: str, network_gates: dict[str, _Gate]) -> None:@@ -855,7 +865,8 @@ def _check_wire_bodies(assert records[path].body == b"" and "body_note" not in records[path].meta[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_network_wire_bodies_remain_readable_after_close(session_type: Any, network_url: str) -> None:with session_type(**_options()) as session:session.fetch(network_url + "/blank")@@ -873,8 +884,9 @@ def test_sync_network_wire_bodies_remain_readable_after_close(session_type: Any,_check_wire_bodies(session.network, wire, native_bodies, native_texts)@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_network_wire_bodies_remain_readable_after_close(session_type: Any, network_url: str) -> None:async with session_type(**_options()) as session:await session.fetch(network_url + "/blank")@@ -912,9 +924,8 @@ def _check_mime_bodies(network: Any, wire: dict[str, list[int]], reads: list[strassert records[path].json() == {"saved": True}- ("session_type", "response_type"), [(DynamicSession, PlaywrightResponse), (StealthySession, PatchrightResponse)]-)[email protected](("session_type", "response_type"), [(StealthySession, PatchrightResponse)])def test_sync_network_reads_only_text_bodies(session_type: Any, response_type: Any, network_url: str, monkeypatch: pytest.MonkeyPatch) -> None:@@ -938,10 +949,11 @@ def test_sync_network_reads_only_text_bodies(_check_mime_bodies(session.network, wire, reads)@pytest.mark.asyncio@pytest.mark.parametrize(("session_type", "response_type"),- [(AsyncDynamicSession, AsyncPlaywrightResponse), (AsyncStealthySession, AsyncPatchrightResponse)],+ [(AsyncStealthySession, AsyncPatchrightResponse)],)async def test_async_network_reads_only_text_bodies(session_type: Any, response_type: Any, network_url: str, monkeypatch: pytest.MonkeyPatch@@ -993,7 +1005,8 @@ def _check_unlimited_search(network: Any, after_id: int) -> None:assert len(network.search(limit=None)) == 125 and len(network.search()) == 100[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_network_unlimited_search_respects_retention(session_type: Any, network_url: str) -> None:with session_type(**_options(max_recorded_requests=125)) as session:session.fetch(network_url + "/blank")@@ -1008,8 +1021,9 @@ def test_sync_network_unlimited_search_respects_retention(session_type: Any, net_check_unlimited_search(session.network, after_id)@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_network_unlimited_search_respects_retention(session_type: Any, network_url: str) -> None:async with session_type(**_options(max_recorded_requests=125)) as session:await session.fetch(network_url + "/blank")diff --git a/tests/fetchers/test_network_config.py b/tests/fetchers/test_network_config.pyindex 585afe2..dc2ba82 100644--- a/tests/fetchers/test_network_config.py+++ b/tests/fetchers/test_network_config.py@@ -2,7 +2,7 @@ import pytestfrom scrapling.core._types import Anyfrom scrapling.engines._browsers._validators import PlaywrightConfig, StealthConfig, validate-from scrapling.fetchers import DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession+from scrapling.fetchers import StealthySession, AsyncStealthySession@pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])@@ -21,7 +21,7 @@ def test_network_config_invalid(model: type[PlaywrightConfig], values: dict[str,validate(values, model)[email protected]("session_type", [DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [StealthySession, AsyncStealthySession])@pytest.mark.parametrize("enabled", [False, True])def test_sessions_configure_one_network_history(session_type: Any, enabled: bool) -> None:session = session_type(record_requests=enabled, max_recorded_requests=7)diff --git a/tests/fetchers/test_public_api.py b/tests/fetchers/test_public_api.pynew file mode 100644index 0000000..c2fd9c6--- /dev/null+++ b/tests/fetchers/test_public_api.py@@ -0,0 +1,51 @@+from importlib import import_module+from importlib.machinery import ModuleSpec+from json import dumps+from pathlib import Path++import pytest++from scrapling.engines.toolbelt import fingerprints++[email protected]("module", ["scrapling", "scrapling.fetchers"])[email protected]("name", ["DynamicFetcher", "DynamicSession", "AsyncDynamicSession", "PlayWrightFetcher"])+def test_dynamic_imports_are_removed(module: str, name: str) -> None:+ package = import_module(module)+ assert name not in package.__all__+ with pytest.raises(ImportError):+ exec(f"from {module} import {name}")++[email protected]("module", ["scrapling.fetchers.chrome", "scrapling.engines._browsers._controllers"])+def test_dynamic_modules_are_removed(module: str) -> None:+ with pytest.raises(ModuleNotFoundError):+ import_module(module)++[email protected]("browser_mode", [True, "chrome"])+def test_browser_version_uses_patchright_when_playwright_is_installed(+ browser_mode: bool | str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ packages: dict[str, ModuleSpec] = {}+ looked_up: list[str] = []+ for name, version in (("patchright", "998.0.0.0"), ("playwright", "999.0.0.0")):+ directory = tmp_path / name+ bundle = directory / "driver" / "package"+ bundle.mkdir(parents=True)+ (bundle / "browsers.json").write_text(dumps({"browsers": [{"name": "chromium", "browserVersion": version}]}))+ packages[name] = ModuleSpec(name, None, origin=str(directory / "__init__.py"))++ def find_spec(name: str) -> ModuleSpec:+ looked_up.append(name)+ return packages[name]++ monkeypatch.setattr(fingerprints, "find_spec", find_spec)+ monkeypatch.setattr(fingerprints, "get_os_name", lambda: "macos")+ fingerprints.driven_browser_version.cache_clear()+ try:+ assert fingerprints.driven_browser_version() == 998+ assert "Chrome/998.0.0.0" in fingerprints.generate_headers(browser_mode=browser_mode)["User-Agent"]+ assert looked_up == ["patchright"]+ finally:+ fingerprints.driven_browser_version.cache_clear()diff --git a/tests/fetchers/test_session_state.py b/tests/fetchers/test_session_state.pyindex 3bb5fce..48a1aa2 100644--- a/tests/fetchers/test_session_state.py+++ b/tests/fetchers/test_session_state.py@@ -6,7 +6,7 @@ import pytestfrom scrapling.core._types import Anyfrom scrapling.engines.toolbelt.proxy_rotation import ProxyRotator-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession+from scrapling.fetchers import AsyncStealthySession, StealthySessionURL = "http://state.test/"@@ -82,7 +82,8 @@ def _check_json(path: Path, native_state: dict[str, Any]) -> None:assert "sessionStorage" not in origin[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_state_round_trip(session_type: Any, tmp_path: Path) -> None:path, native_path = tmp_path / "state.json", tmp_path / "native.json"with session_type(**_options()) as source:@@ -114,8 +115,9 @@ def test_sync_state_round_trip(session_type: Any, tmp_path: Path) -> None:_check_json(path, loads(native_path.read_text()))@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_state_round_trip(session_type: Any, tmp_path: Path) -> None:async def route_page(route: Any) -> None:await route.fulfill(body=HTML, content_type="text/html")@@ -150,7 +152,8 @@ async def test_async_state_round_trip(session_type: Any, tmp_path: Path) -> None_check_json(path, loads(native_path.read_text()))[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_state_file_errors(session_type: Any, tmp_path: Path) -> None:invalid = tmp_path / "invalid.json"invalid.write_text("{")@@ -166,8 +169,9 @@ def test_sync_state_file_errors(session_type: Any, tmp_path: Path) -> None:session.save_state("")@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_state_file_errors(session_type: Any, tmp_path: Path) -> None:invalid = tmp_path / "invalid.json"invalid.write_text("{")@@ -183,7 +187,7 @@ async def test_async_state_file_errors(session_type: Any, tmp_path: Path) -> Nonawait session.save_state("")[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_state_before_start(session_type: Any, tmp_path: Path) -> None:session = session_type(**_options())for method in (session.save_state, session.load_state):@@ -192,7 +196,7 @@ def test_sync_state_before_start(session_type: Any, tmp_path: Path) -> None:@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_state_before_start(session_type: Any, tmp_path: Path) -> None:session = session_type(**_options())for method in (session.save_state, session.load_state):@@ -200,7 +204,8 @@ async def test_async_state_before_start(session_type: Any, tmp_path: Path) -> Noawait method(tmp_path / "state.json")[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_state_with_proxy_rotation(session_type: Any, tmp_path: Path) -> None:with session_type(**_options(proxy_rotator=ProxyRotator(["http://127.0.0.1:1"]))) as session:for method in (session.save_state, session.load_state):@@ -208,8 +213,9 @@ def test_sync_state_with_proxy_rotation(session_type: Any, tmp_path: Path) -> Nomethod(tmp_path / "state.json")@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_state_with_proxy_rotation(session_type: Any, tmp_path: Path) -> None:async with session_type(**_options(proxy_rotator=ProxyRotator(["http://127.0.0.1:1"]))) as session:for method in (session.save_state, session.load_state):diff --git a/tests/fetchers/test_shadow_browser.py b/tests/fetchers/test_shadow_browser.pyindex 2a543d7..3e7c2a8 100644--- a/tests/fetchers/test_shadow_browser.py+++ b/tests/fetchers/test_shadow_browser.py@@ -4,7 +4,7 @@ import pytestfrom scrapling.core._types import Anyfrom scrapling.engines.toolbelt.custom import Response-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession+from scrapling.fetchers import AsyncStealthySession, StealthySessionHTML = """<!DOCTYPE html><html><head><title>Shadow test</title></head><body>@@ -48,7 +48,8 @@ async def setup_async_page(page: Any) -> None:await page.route("**/*", lambda route: route.fulfill(status=200, content_type="text/html", body=HTML))[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_shadow_browser(session_type: Any) -> None:with session_type(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"),@@ -60,8 +61,9 @@ def test_sync_shadow_browser(session_type: Any) -> None:assert not session.fetch("https://shadow.test/", pierce_shadow=False).css("#shadow")@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_shadow_browser(session_type: Any) -> None:async with session_type(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"),diff --git a/tests/fetchers/test_shadow_hidden_slots.py b/tests/fetchers/test_shadow_hidden_slots.pyindex 633724d..0eb9b30 100644--- a/tests/fetchers/test_shadow_hidden_slots.py+++ b/tests/fetchers/test_shadow_hidden_slots.py@@ -25,6 +25,7 @@ def check_clean_output(response: Response, visible: tuple[str, ...], hidden: tupassert all(value not in output for value in hidden)@pytest.mark.asyncio@pytest.mark.parametrize(("attrs", "key", "value"), HIDDEN_ATTRIBUTES)async def test_hidden_slots_keep_attributes_and_assigned_content(attrs: str, key: str, value: str) -> None:@@ -66,6 +67,7 @@ async def test_hidden_slots_keep_attributes_and_assigned_content(attrs: str, keyassert response.body.decode() == body@pytest.mark.asyncio@pytest.mark.parametrize("assigned", [False, True])@pytest.mark.parametrize(("attrs", "key", "value"), HIDDEN_ATTRIBUTES)diff --git a/tests/fetchers/test_shadow_order.py b/tests/fetchers/test_shadow_order.pyindex 95b0f87..cf0beda 100644--- a/tests/fetchers/test_shadow_order.py+++ b/tests/fetchers/test_shadow_order.py@@ -2,7 +2,7 @@ from os import getenvfrom pathlib import Pathimport pytest-from playwright.async_api import async_playwright+from patchright.async_api import async_playwrightfrom scrapling.engines.toolbelt.convertor import ResponseFactoryfrom scrapling.engines.toolbelt.custom import Response@@ -42,6 +42,7 @@ async def extract(html: str, setup: str) -> Response:await browser.close()@pytest.mark.asyncioasync def test_shadow_slots_follow_composed_order(tmp_path: Path) -> None:response = await extract(@@ -81,6 +82,7 @@ async def test_shadow_slots_follow_composed_order(tmp_path: Path) -> None:assert not restored.css("#unused")@pytest.mark.asyncio@pytest.mark.parametrize("assigned", [False, True])async def test_shadow_forwarded_slots_and_nested_hosts(assigned: bool) -> None:diff --git a/tests/fetchers/test_shadow_roundtrip.py b/tests/fetchers/test_shadow_roundtrip.pyindex c24250b..55a207a 100644--- a/tests/fetchers/test_shadow_roundtrip.py+++ b/tests/fetchers/test_shadow_roundtrip.py@@ -1,7 +1,7 @@from os import getenvimport pytest-from playwright.async_api import async_playwright+from patchright.async_api import async_playwrightfrom scrapling.core._types import extraction_typesfrom scrapling.core.shell import Convertor@@ -26,6 +26,7 @@ def check_roundtrip(content: bytes) -> None:assert response.body == content@pytest.mark.asyncioasync def test_shadow_wrappers_roundtrip_across_host_tags() -> None:async with async_playwright() as driver:diff --git a/tests/fetchers/test_shadow_snapshot.py b/tests/fetchers/test_shadow_snapshot.pyindex c2d5da2..b19ca05 100644--- a/tests/fetchers/test_shadow_snapshot.py+++ b/tests/fetchers/test_shadow_snapshot.py@@ -1,11 +1,12 @@from os import getenvimport pytest-from playwright.async_api import async_playwright+from patchright.async_api import async_playwrightfrom scrapling.engines.toolbelt.convertor import SHADOW_SNAPSHOT_JS, ResponseFactory@pytest.mark.asyncioasync def test_shadow_snapshot_preserves_source_content() -> None:async with async_playwright() as driver:@@ -25,7 +26,8 @@ async def test_shadow_snapshot_preserves_source_content() -> None:baseline = await page.content()plain = await ResponseFactory.from_async_playwright_response(page, first, None, {}, pierce_shadow=True)assert plain.body.decode() == baseline- await page.evaluate("""() => {+ await page.evaluate(+ """() => {const root = document.querySelector('#host').attachShadow({mode: 'open'});root.innerHTML = '<div id="inside">SHADOW<slot name="label">WRONG-FALLBACK</slot><slot name="missing">USED-FALLBACK</slot><div id="nested"></div><svg viewBox="0 0 10 10"><path d="M0 0"/></svg></div>';let host = root.querySelector('#nested');@@ -73,17 +75,19 @@ async def test_shadow_snapshot_preserves_source_content() -> None:root.append(scopedHost);scopedHost.attachShadow({mode: 'open', customElementRegistry: registry}).innerHTML = '<shadow-scoped-counter>SCOPED-VALUE</shadow-scoped-counter>';}- }""")+ }""",+ isolated_context=False,+ )before = await page.content()shadow_before = await page.locator("#host").evaluate("node => node.shadowRoot.innerHTML")- count = await page.evaluate("globalThis.shadowConstructed")+ count = await page.evaluate("globalThis.shadowConstructed", isolated_context=False)noscript = await page.locator("#noscript").evaluate("node => node.outerHTML")response = await ResponseFactory.from_async_playwright_response(page, first, None, {"keep_comments": True}, pierce_shadow=True)assert await page.content() == beforeassert await page.locator("#host").evaluate("node => node.shadowRoot.innerHTML") == shadow_before- assert await page.evaluate("globalThis.shadowConstructed") == count+ assert await page.evaluate("globalThis.shadowConstructed", isolated_context=False) == countassert response.css("#host > shadow-root > #inside").get()assert all(response.css(f"#level-{i}::text").get() == f"LEVEL-{i}" for i in range(4))assert response.css("#data::text").get() == '{"name":"A&B<C>","title":"Bürger"}'@@ -106,7 +110,7 @@ async def test_shadow_snapshot_preserves_source_content() -> None:assert "LEVEL-3" in markdownassert "INACTIVE" not in markdownassert "FAKE-SHADOW" not in markdown- if await page.evaluate("typeof CustomElementRegistry === 'function'"):+ if await page.evaluate("typeof CustomElementRegistry === 'function'", isolated_context=False):assert response.css("shadow-scoped-counter::text").get() == "SCOPED-VALUE"await page.set_content('<!DOCTYPE html><html><body><template id="only"><div></div></template></body></html>'diff --git a/tests/fetchers/test_snapshot.py b/tests/fetchers/test_snapshot.pyindex 0e520a3..a05de63 100644--- a/tests/fetchers/test_snapshot.py+++ b/tests/fetchers/test_snapshot.py@@ -5,7 +5,7 @@ from unittest.mock import AsyncMock, Mockimport pytestfrom scrapling.core._types import Any-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession+from scrapling.fetchers import AsyncStealthySession, StealthySessionHTML = """<!DOCTYPE html><html><body>@@ -15,7 +15,7 @@ HTML = """<!DOCTYPE html><html><body></body></html>"""[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])@pytest.mark.parametrize("css_selector", [None, "main"])def test_sync_snapshot_forwards_options_and_errors(session_type: Any, css_selector: str | None) -> None:page = Mock()@@ -34,7 +34,7 @@ def test_sync_snapshot_forwards_options_and_errors(session_type: Any, css_select@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])@pytest.mark.parametrize("css_selector", [None, "main"])async def test_async_snapshot_forwards_options_and_errors(session_type: Any, css_selector: str | None) -> None:page = Mock()@@ -61,7 +61,8 @@ def _button_ref(snapshot: str) -> str:return match[1][email protected]("session_type", [DynamicSession, StealthySession])[email protected]("session_type", [StealthySession])def test_sync_snapshot_live_state_and_refs(session_type: Any) -> None:def setup(page: Any) -> None:page.route("**/*", lambda route: route.fulfill(status=200, content_type="text/html", body=HTML))@@ -83,8 +84,9 @@ def test_sync_snapshot_live_state_and_refs(session_type: Any) -> None:assert session.page_pool.pages_count == 1@pytest.mark.asyncio[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("session_type", [AsyncStealthySession])async def test_async_snapshot_live_state_and_refs(session_type: Any) -> None:async def setup(page: Any) -> None:await page.route("**/*", lambda route: route.fulfill(status=200, content_type="text/html", body=HTML))diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.pyindex 8f36672..14e125f 100644--- a/tests/fetchers/test_utils.py+++ b/tests/fetchers/test_utils.py@@ -260,7 +260,7 @@ class TestFingerprintFunctions:def test_driven_browser_version(self):"""Test that the driven Chromium version is read from the installed automation package"""- assert isinstance(driven_browser_version("playwright"), int)+ assert isinstance(driven_browser_version(), int)assert driven_browser_version("not_a_real_package") is None@pytest.mark.parametrize("browser_mode", [True, "chrome"])diff --git a/tests/fetchers/test_validator.py b/tests/fetchers/test_validator.pyindex 3827194..71e8068 100644--- a/tests/fetchers/test_validator.py+++ b/tests/fetchers/test_validator.py@@ -1,7 +1,7 @@import pytestfrom types import SimpleNamespace-from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession+from scrapling.fetchers import AsyncStealthySession, StealthySessionfrom scrapling.engines._browsers._validators import (validate,validate_fetch,@@ -37,9 +37,7 @@ class TestValidators:assert params.timeout == config.timeoutassert not hasattr(params, "unknown_option")- @pytest.mark.parametrize(- "session_type", [DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession]- )+ @pytest.mark.parametrize("session_type", [StealthySession, AsyncStealthySession])def test_removed_xhr_option_is_rejected_by_sessions(self, session_type):with pytest.raises(TypeError, match="capture_xhr was removed"):session_type(capture_xhr=".*")diff --git a/tests/spiders/test_browser_session.py b/tests/spiders/test_browser_session.pynew file mode 100644index 0000000..df32a79--- /dev/null+++ b/tests/spiders/test_browser_session.py@@ -0,0 +1,29 @@+from os import getenv++import pytest++from scrapling.core._types import Any+from scrapling.fetchers import AsyncStealthySession+from scrapling.spiders import Request, SessionManager++[email protected]("lazy", [False, True])+async def test_stealthy_browser_session_dispatch(lazy: bool) -> None:+ async def setup(page: Any) -> None:+ await page.route("**/*", lambda route: route.fulfill(content_type="text/html", body="<h1>Browser</h1>"))++ session = AsyncStealthySession(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"), google_search=False)+ manager = SessionManager().add("browser", session, lazy=lazy)+ request = Request("https://spider.test/", sid="browser", page_setup=setup, meta={"source": "browser"})+ async with manager:+ assert session._is_alive is not lazy+ response = await manager.fetch(request)+ assert response.status == 200+ assert response.css("h1::text").get() == "Browser"+ assert response.request is request+ assert response.meta["source"] == "browser"+ assert session._is_alive+ assert not session._is_alive+ assert session.page_pool.pages_count == 0diff --git a/tox.ini b/tox.iniindex 0e91178..7ca4e6a 100644--- a/tox.ini+++ b/tox.ini@@ -16,14 +16,14 @@ deps =extras = ai,shellcommands =# Run browser tests without parallelization (avoid browser conflicts)- pytest --cov=scrapling --cov-report=xml -k "DynamicFetcher or StealthyFetcher" --verbose+ pytest --cov=scrapling --cov-report=xml -m "browser" --verbose# Run asyncio tests without parallelization (avoid GitHub CI nested loop issues)- pytest --cov=scrapling --cov-report=xml -m "asyncio" -k "not (DynamicFetcher or StealthyFetcher)" --verbose --cov-append+ pytest --cov=scrapling --cov-report=xml -m "asyncio and not browser" --verbose --cov-append# Run everything else with parallelization (for speed)- pytest --cov=scrapling --cov-report=xml -m "not asyncio" -k "not (DynamicFetcher or StealthyFetcher)" -n auto --cov-append+ pytest --cov=scrapling --cov-report=xml -m "not asyncio and not browser" -n auto --cov-append[testenv:pre-commit]basepython = python3deps = pre-commitcommands = pre-commit run --all-files --show-diff-on-failure-skip_install = true\ No newline at end of file+skip_install = true