Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

test: add tests for the new `impersonate` logic

Karim shoair275165319036
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index 1f98bfb..4a79c25 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -193,3 +193,51 @@ class TestCLI:
[html_url, 'output.invalid']
)
# Should handle the error gracefully
+
+ def test_impersonate_comma_separated(self, runner, tmp_path, html_url):
+ """Test that comma-separated impersonate values are parsed correctly"""
+ output_file = tmp_path / "output.md"
+
+ with patch('scrapling.fetchers.Fetcher.get') as mock_get:
+ mock_response = configure_selector_mock()
+ mock_response.status = 200
+ mock_get.return_value = mock_response
+
+ result = runner.invoke(
+ get,
+ [
+ html_url,
+ str(output_file),
+ '--impersonate', 'chrome,firefox,safari'
+ ]
+ )
+ assert result.exit_code == 0
+
+ # Verify that the impersonate argument was converted to a list
+ call_kwargs = mock_get.call_args[1]
+ assert isinstance(call_kwargs['impersonate'], list)
+ assert call_kwargs['impersonate'] == ['chrome', 'firefox', 'safari']
+
+ def test_impersonate_single_browser(self, runner, tmp_path, html_url):
+ """Test that single impersonate value remains as string"""
+ output_file = tmp_path / "output.md"
+
+ with patch('scrapling.fetchers.Fetcher.get') as mock_get:
+ mock_response = configure_selector_mock()
+ mock_response.status = 200
+ mock_get.return_value = mock_response
+
+ result = runner.invoke(
+ get,
+ [
+ html_url,
+ str(output_file),
+ '--impersonate', 'chrome'
+ ]
+ )
+ assert result.exit_code == 0
+
+ # Verify that the impersonate argument remains a string
+ call_kwargs = mock_get.call_args[1]
+ assert isinstance(call_kwargs['impersonate'], str)
+ assert call_kwargs['impersonate'] == 'chrome'
diff --git a/tests/fetchers/test_impersonate_list.py b/tests/fetchers/test_impersonate_list.py
new file mode 100644
index 0000000..d79a06b
--- /dev/null
+++ b/tests/fetchers/test_impersonate_list.py
@@ -0,0 +1,153 @@
+"""Test suite for list-based impersonate parameter functionality."""
+import pytest
+import pytest_httpbin
+from unittest.mock import patch, MagicMock
+
+from scrapling import Fetcher
+from scrapling.fetchers import FetcherSession
+from scrapling.engines.static import _select_random_browser
+
+
+class TestRandomBrowserSelection:
+ """Test the random browser selection helper function."""
+
+ def test_select_random_browser_with_single_string(self):
+ """Test that single browser string is returned as-is."""
+ result = _select_random_browser("chrome")
+ assert result == "chrome"
+
+ def test_select_random_browser_with_none(self):
+ """Test that None is returned as-is."""
+ result = _select_random_browser(None)
+ assert result is None
+
+ def test_select_random_browser_with_list(self):
+ """Test that a browser is randomly selected from a list."""
+ browsers = ["chrome", "firefox", "safari"]
+ result = _select_random_browser(browsers)
+ assert result in browsers
+
+ def test_select_random_browser_with_empty_list(self):
+ """Test that empty list returns None."""
+ result = _select_random_browser([])
+ assert result is None
+
+ def test_select_random_browser_with_single_item_list(self):
+ """Test that single-item list returns that item."""
+ result = _select_random_browser(["chrome"])
+ assert result == "chrome"
+
+
+@pytest_httpbin.use_class_based_httpbin
+class TestFetcherWithImpersonateList:
+ """Test Fetcher with list-based impersonate parameter."""
+
+ @pytest.fixture(autouse=True)
+ def setup_urls(self, httpbin):
+ """Fixture to set up URLs for testing."""
+ self.basic_url = f"{httpbin.url}/get"
+
+ def test_get_with_impersonate_list(self):
+ """Test that GET request works with impersonate as a list."""
+ browsers = ["chrome", "firefox"]
+ response = Fetcher.get(self.basic_url, impersonate=browsers)
+ assert response.status == 200
+
+ def test_get_with_single_impersonate(self):
+ """Test that GET request still works with single browser string."""
+ response = Fetcher.get(self.basic_url, impersonate="chrome")
+ assert response.status == 200
+
+ def test_post_with_impersonate_list(self):
+ """Test that POST request works with impersonate as a list."""
+ browsers = ["chrome", "firefox"]
+ post_url = self.basic_url.replace("/get", "/post")
+ response = Fetcher.post(post_url, data={"key": "value"}, impersonate=browsers)
+ assert response.status == 200
+
+ def test_put_with_impersonate_list(self):
+ """Test that PUT request works with impersonate as a list."""
+ browsers = ["chrome", "safari"]
+ put_url = self.basic_url.replace("/get", "/put")
+ response = Fetcher.put(put_url, data={"key": "value"}, impersonate=browsers)
+ assert response.status == 200
+
+ def test_delete_with_impersonate_list(self):
+ """Test that DELETE request works with impersonate as a list."""
+ browsers = ["chrome", "edge"]
+ delete_url = self.basic_url.replace("/get", "/delete")
+ response = Fetcher.delete(delete_url, impersonate=browsers)
+ assert response.status == 200
+
+
+@pytest_httpbin.use_class_based_httpbin
+class TestFetcherSessionWithImpersonateList:
+ """Test FetcherSession with list-based impersonate parameter."""
+
+ @pytest.fixture(autouse=True)
+ def setup_urls(self, httpbin):
+ """Fixture to set up URLs for testing."""
+ self.basic_url = f"{httpbin.url}/get"
+
+ def test_session_init_with_impersonate_list(self):
+ """Test that FetcherSession can be initialized with impersonate as a list."""
+ browsers = ["chrome", "firefox", "safari"]
+ session = FetcherSession(impersonate=browsers)
+ assert session._default_impersonate == browsers
+
+ def test_session_request_with_impersonate_list(self):
+ """Test that session request works with impersonate as a list."""
+ browsers = ["chrome", "firefox"]
+ with FetcherSession(impersonate=browsers) as session:
+ response = session.get(self.basic_url)
+ assert response.status == 200
+
+ def test_session_multiple_requests_with_impersonate_list(self):
+ """Test that multiple requests in a session work with impersonate list."""
+ browsers = ["chrome110", "chrome120", "chrome131"]
+ with FetcherSession(impersonate=browsers) as session:
+ response1 = session.get(self.basic_url)
+ response2 = session.get(self.basic_url)
+ assert response1.status == 200
+ assert response2.status == 200
+
+ def test_session_request_level_impersonate_override(self):
+ """Test that request-level impersonate overrides session-level."""
+ session_browsers = ["chrome", "firefox"]
+ request_browser = "safari"
+
+ with FetcherSession(impersonate=session_browsers) as session:
+ response = session.get(self.basic_url, impersonate=request_browser)
+ assert response.status == 200
+
+ def test_session_request_level_impersonate_list_override(self):
+ """Test that request-level impersonate list overrides session-level."""
+ session_browsers = ["chrome", "firefox"]
+ request_browsers = ["safari", "edge"]
+
+ with FetcherSession(impersonate=session_browsers) as session:
+ response = session.get(self.basic_url, impersonate=request_browsers)
+ assert response.status == 200
+
+
+class TestImpersonateTypeValidation:
+ """Test type validation for impersonate parameter."""
+
+ def test_impersonate_accepts_string(self):
+ """Test that impersonate accepts string type."""
+ # This should not raise any type errors
+ session = FetcherSession(impersonate="chrome")
+ assert session._default_impersonate == "chrome"
+
+ def test_impersonate_accepts_list(self):
+ """Test that impersonate accepts list type."""
+ # This should not raise any type errors
+ browsers = ["chrome", "firefox"]
+ session = FetcherSession(impersonate=browsers)
+ assert session._default_impersonate == browsers
+
+ def test_impersonate_accepts_none(self):
+ """Test that impersonate accepts None."""
+ # This should not raise any type errors
+ session = FetcherSession(impersonate=None)
+ assert session._default_impersonate is None