Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

test: add new tests for the proxy rotation logic

Karim shoair6e0277f23b99
diff --git a/tests/fetchers/test_proxy_rotation.py b/tests/fetchers/test_proxy_rotation.py
new file mode 100644
index 0000000..d418c1a
--- /dev/null
+++ b/tests/fetchers/test_proxy_rotation.py
@@ -0,0 +1,303 @@
+import pytest
+import random
+from threading import Thread
+from concurrent.futures import ThreadPoolExecutor
+
+from scrapling.engines.toolbelt import ProxyRotator, is_proxy_error, round_robin
+
+
+class TestRoundRobinStrategy:
+ """Test the default round_robin strategy function"""
+
+ def test_round_robin_cycles_through_proxies(self):
+ """Test that round_robin returns proxies in order"""
+ proxies = ["http://p1:8080", "http://p2:8080", "http://p3:8080"]
+
+ proxy, next_idx = round_robin(proxies, 0)
+ assert proxy == "http://p1:8080"
+ assert next_idx == 1
+
+ proxy, next_idx = round_robin(proxies, 1)
+ assert proxy == "http://p2:8080"
+ assert next_idx == 2
+
+ proxy, next_idx = round_robin(proxies, 2)
+ assert proxy == "http://p3:8080"
+ assert next_idx == 0 # Wraps around
+
+ def test_round_robin_wraps_index(self):
+ """Test that round_robin handles index overflow"""
+ proxies = ["http://p1:8080", "http://p2:8080"]
+
+ # Index larger than list length should wrap
+ proxy, next_idx = round_robin(proxies, 5)
+ assert proxy == "http://p2:8080" # 5 % 2 = 1
+ assert next_idx == 0
+
+
+class TestProxyRotatorCreation:
+ """Test ProxyRotator initialization and validation"""
+
+ def test_create_with_string_proxies(self):
+ """Test creating rotator with string proxy URLs"""
+ proxies = ["http://p1:8080", "http://p2:8080"]
+ rotator = ProxyRotator(proxies)
+
+ assert len(rotator) == 2
+ assert rotator.proxies == proxies
+
+ def test_create_with_dict_proxies(self):
+ """Test creating rotator with dict proxies"""
+ proxies = [
+ {"server": "http://p1:8080", "username": "user1", "password": "pass1"},
+ {"server": "http://p2:8080"},
+ ]
+ rotator = ProxyRotator(proxies)
+
+ assert len(rotator) == 2
+ assert rotator.proxies == proxies
+
+ def test_create_with_mixed_proxies(self):
+ """Test creating rotator with mixed string and dict proxies"""
+ proxies = [
+ "http://p1:8080",
+ {"server": "http://p2:8080", "username": "user"},
+ ]
+ rotator = ProxyRotator(proxies)
+
+ assert len(rotator) == 2
+
+ def test_empty_proxies_raises_error(self):
+ """Test that empty proxy list raises ValueError"""
+ with pytest.raises(ValueError, match="At least one proxy must be provided"):
+ ProxyRotator([])
+
+ def test_dict_without_server_raises_error(self):
+ """Test that dict proxy without 'server' key raises ValueError"""
+ with pytest.raises(ValueError, match="Proxy dict must have a 'server' key"):
+ ProxyRotator([{"username": "user", "password": "pass"}])
+
+ def test_invalid_proxy_type_raises_error(self):
+ """Test that invalid proxy type raises TypeError"""
+ with pytest.raises(TypeError, match="Invalid proxy type"):
+ ProxyRotator([123])
+
+ with pytest.raises(TypeError, match="Invalid proxy type"):
+ ProxyRotator([None])
+
+ def test_non_callable_strategy_raises_error(self):
+ """Test that non-callable strategy raises TypeError"""
+ with pytest.raises(TypeError, match="strategy must be callable"):
+ ProxyRotator(["http://p1:8080"], strategy="round_robin")
+
+ with pytest.raises(TypeError, match="strategy must be callable"):
+ ProxyRotator(["http://p1:8080"], strategy=123)
+
+
+class TestProxyRotatorRotation:
+ """Test ProxyRotator rotation behavior"""
+
+ def test_get_proxy_round_robin(self):
+ """Test that get_proxy cycles through proxies in order"""
+ proxies = ["http://p1:8080", "http://p2:8080", "http://p3:8080"]
+ rotator = ProxyRotator(proxies)
+
+ # First cycle
+ assert rotator.get_proxy() == "http://p1:8080"
+ assert rotator.get_proxy() == "http://p2:8080"
+ assert rotator.get_proxy() == "http://p3:8080"
+
+ # Second cycle - wraps around
+ assert rotator.get_proxy() == "http://p1:8080"
+ assert rotator.get_proxy() == "http://p2:8080"
+ assert rotator.get_proxy() == "http://p3:8080"
+
+ def test_get_proxy_single_proxy(self):
+ """Test rotation with single proxy always returns the same proxy"""
+ rotator = ProxyRotator(["http://only:8080"])
+
+ for _ in range(5):
+ assert rotator.get_proxy() == "http://only:8080"
+
+ def test_get_proxy_with_dict_proxies(self):
+ """Test rotation with dict proxies"""
+ proxies = [
+ {"server": "http://p1:8080"},
+ {"server": "http://p2:8080"},
+ ]
+ rotator = ProxyRotator(proxies)
+
+ assert rotator.get_proxy() == {"server": "http://p1:8080"}
+ assert rotator.get_proxy() == {"server": "http://p2:8080"}
+ assert rotator.get_proxy() == {"server": "http://p1:8080"}
+
+
+class TestCustomStrategies:
+ """Test ProxyRotator with custom rotation strategies"""
+
+ def test_random_strategy(self):
+ """Test custom random selection strategy"""
+ def random_strategy(proxies, idx):
+ return random.choice(proxies), idx
+
+ proxies = ["http://p1:8080", "http://p2:8080", "http://p3:8080"]
+ rotator = ProxyRotator(proxies, strategy=random_strategy)
+
+ # Get multiple proxies - they should all be valid
+ results = [rotator.get_proxy() for _ in range(10)]
+ assert all(p in proxies for p in results)
+
+ def test_sticky_strategy(self):
+ """Test custom sticky strategy that always returns first proxy"""
+ def sticky_strategy(proxies, idx):
+ return proxies[0], idx
+
+ rotator = ProxyRotator(
+ ["http://p1:8080", "http://p2:8080"],
+ strategy=sticky_strategy
+ )
+
+ for _ in range(5):
+ assert rotator.get_proxy() == "http://p1:8080"
+
+ def test_weighted_strategy(self):
+ """Test custom weighted strategy"""
+ call_count = {"count": 0}
+
+ def alternating_strategy(proxies, idx):
+ # Returns first proxy twice, then second proxy once
+ call_count["count"] += 1
+ if call_count["count"] % 3 == 0:
+ return proxies[1], idx
+ return proxies[0], idx
+
+ rotator = ProxyRotator(
+ ["http://primary:8080", "http://backup:8080"],
+ strategy=alternating_strategy
+ )
+
+ assert rotator.get_proxy() == "http://primary:8080"
+ assert rotator.get_proxy() == "http://primary:8080"
+ assert rotator.get_proxy() == "http://backup:8080"
+
+ def test_lambda_strategy(self):
+ """Test using lambda as strategy"""
+ rotator = ProxyRotator(
+ ["http://p1:8080", "http://p2:8080"],
+ strategy=lambda proxies, idx: (proxies[-1], idx) # Always last
+ )
+
+ assert rotator.get_proxy() == "http://p2:8080"
+ assert rotator.get_proxy() == "http://p2:8080"
+
+
+class TestProxyRotatorProperties:
+ """Test ProxyRotator properties and methods"""
+
+ def test_proxies_property_returns_copy(self):
+ """Test that proxies property returns a copy, not the original list"""
+ original = ["http://p1:8080", "http://p2:8080"]
+ rotator = ProxyRotator(original)
+
+ proxies_copy = rotator.proxies
+ proxies_copy.append("http://p3:8080")
+
+ # Original should be unchanged
+ assert len(rotator) == 2
+ assert len(rotator.proxies) == 2
+
+ def test_len_returns_proxy_count(self):
+ """Test __len__ returns correct count"""
+ assert len(ProxyRotator(["http://p1:8080"])) == 1
+ assert len(ProxyRotator(["http://p1:8080", "http://p2:8080"])) == 2
+ assert len(ProxyRotator(["a", "b", "c", "d", "e"])) == 5
+
+ def test_repr(self):
+ """Test __repr__ format"""
+ rotator = ProxyRotator(["http://p1:8080", "http://p2:8080", "http://p3:8080"])
+ assert repr(rotator) == "ProxyRotator(proxies=3)"
+
+
+class TestProxyRotatorThreadSafety:
+ """Test ProxyRotator thread safety"""
+
+ def test_concurrent_get_proxy(self):
+ """Test that concurrent get_proxy calls don't cause errors"""
+ proxies = [f"http://p{i}:8080" for i in range(10)]
+ rotator = ProxyRotator(proxies)
+ results = []
+
+ def get_proxies(n):
+ for _ in range(n):
+ results.append(rotator.get_proxy())
+
+ threads = [Thread(target=get_proxies, args=(100,)) for _ in range(10)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+
+ # All results should be valid proxies
+ assert len(results) == 1000
+ assert all(p in proxies for p in results)
+
+ def test_thread_pool_concurrent_access(self):
+ """Test concurrent access using ThreadPoolExecutor"""
+ proxies = ["http://p1:8080", "http://p2:8080", "http://p3:8080"]
+ rotator = ProxyRotator(proxies)
+
+ with ThreadPoolExecutor(max_workers=5) as executor:
+ futures = [executor.submit(rotator.get_proxy) for _ in range(100)]
+ results = [f.result() for f in futures]
+
+ assert len(results) == 100
+ assert all(p in proxies for p in results)
+
+
+class TestIsProxyError:
+ """Test is_proxy_error utility function"""
+
+ @pytest.mark.parametrize("error_msg", [
+ "net::err_proxy_connection_failed",
+ "NET::ERR_PROXY_AUTH_FAILED",
+ "net::err_tunnel_connection_failed",
+ "Connection refused by proxy",
+ "Connection reset by peer",
+ "Connection timed out while connecting to proxy",
+ "Failed to connect to proxy server",
+ "Could not resolve proxy host",
+ ])
+ def test_proxy_errors_detected(self, error_msg):
+ """Test that proxy-related errors are detected"""
+ assert is_proxy_error(Exception(error_msg)) is True
+
+ @pytest.mark.parametrize("error_msg", [
+ "Page not found",
+ "404 Not Found",
+ "Internal server error",
+ "DNS resolution failed",
+ "SSL certificate error",
+ "Timeout waiting for response",
+ "Invalid JSON response",
+ ])
+ def test_non_proxy_errors_not_detected(self, error_msg):
+ """Test that non-proxy errors are not detected as proxy errors"""
+ assert is_proxy_error(Exception(error_msg)) is False
+
+ def test_case_insensitive_detection(self):
+ """Test that error detection is case-insensitive"""
+ assert is_proxy_error(Exception("NET::ERR_PROXY")) is True
+ assert is_proxy_error(Exception("Net::Err_Proxy")) is True
+ assert is_proxy_error(Exception("CONNECTION REFUSED")) is True
+
+ def test_empty_error_message(self):
+ """Test handling of empty error message"""
+ assert is_proxy_error(Exception("")) is False
+
+ def test_custom_exception_types(self):
+ """Test with custom exception types"""
+ class CustomError(Exception):
+ pass
+
+ assert is_proxy_error(CustomError("net::err_proxy_failed")) is True
+ assert is_proxy_error(CustomError("normal error")) is False