🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: use more accurate naming for proxy rotation strategy
Karim shoair0ab54852c47a
diff --git a/scrapling/engines/toolbelt/__init__.py b/scrapling/engines/toolbelt/__init__.pyindex f3714c7..fea2781 100644--- a/scrapling/engines/toolbelt/__init__.py+++ b/scrapling/engines/toolbelt/__init__.py@@ -1,3 +1,3 @@-from .proxy_rotation import ProxyRotator, is_proxy_error, round_robin+from .proxy_rotation import ProxyRotator, is_proxy_error, cyclic_rotation-__all__ = ["ProxyRotator", "is_proxy_error", "round_robin"]+__all__ = ["ProxyRotator", "is_proxy_error", "cyclic_rotation"]diff --git a/scrapling/engines/toolbelt/proxy_rotation.py b/scrapling/engines/toolbelt/proxy_rotation.pyindex 82c270d..37cdba1 100644--- a/scrapling/engines/toolbelt/proxy_rotation.py+++ b/scrapling/engines/toolbelt/proxy_rotation.py@@ -30,8 +30,8 @@ def is_proxy_error(error: Exception) -> bool:return any(indicator in error_msg for indicator in _PROXY_ERROR_INDICATORS)-def round_robin(proxies: List[ProxyType], current_index: int) -> Tuple[ProxyType, int]:- """Default round-robin rotation strategy."""+def cyclic_rotation(proxies: List[ProxyType], current_index: int) -> Tuple[ProxyType, int]:+ """Default cyclic rotation strategy — iterates through proxies sequentially, wrapping around at the end."""idx = current_index % len(proxies)return proxies[idx], (idx + 1) % len(proxies)@@ -41,7 +41,7 @@ class ProxyRotator:A thread-safe proxy rotator with pluggable rotation strategies.Supports:- - Round-robin rotation (default)+ - Cyclic rotation (default)- Custom rotation strategies via callable- Both string URLs and Playwright-style dict proxies"""@@ -51,7 +51,7 @@ class ProxyRotator:def __init__(self,proxies: List[ProxyType],- strategy: RotationStrategy = round_robin,+ strategy: RotationStrategy = cyclic_rotation,):"""Initialize the proxy rotator.@@ -59,7 +59,7 @@ class ProxyRotator::param proxies: List of proxy URLs or Playwright-style proxy dicts.- String format: "http://proxy1:8080" or "http://user:pass@proxy:8080"- Dict format: {"server": "http://proxy:8080", "username": "user", "password": "pass"}- :param strategy: Rotation strategy function. Takes (proxies, current_index) and returns (proxy, next_index). Defaults to round_robin.+ :param strategy: Rotation strategy function. Takes (proxies, current_index) and returns (proxy, next_index). Defaults to cyclic_rotation."""if not proxies:raise ValueError("At least one proxy must be provided")diff --git a/tests/fetchers/test_proxy_rotation.py b/tests/fetchers/test_proxy_rotation.pyindex d418c1a..e08104d 100644--- a/tests/fetchers/test_proxy_rotation.py+++ b/tests/fetchers/test_proxy_rotation.py@@ -3,34 +3,34 @@ import randomfrom threading import Threadfrom concurrent.futures import ThreadPoolExecutor-from scrapling.engines.toolbelt import ProxyRotator, is_proxy_error, round_robin+from scrapling.engines.toolbelt import ProxyRotator, is_proxy_error, cyclic_rotation-class TestRoundRobinStrategy:- """Test the default round_robin strategy function"""+class TestCyclicRotationStrategy:+ """Test the default cyclic_rotation strategy function"""- def test_round_robin_cycles_through_proxies(self):- """Test that round_robin returns proxies in order"""+ def test_cyclic_rotation_cycles_through_proxies(self):+ """Test that cyclic_rotation returns proxies in order"""proxies = ["http://p1:8080", "http://p2:8080", "http://p3:8080"]- proxy, next_idx = round_robin(proxies, 0)+ proxy, next_idx = cyclic_rotation(proxies, 0)assert proxy == "http://p1:8080"assert next_idx == 1- proxy, next_idx = round_robin(proxies, 1)+ proxy, next_idx = cyclic_rotation(proxies, 1)assert proxy == "http://p2:8080"assert next_idx == 2- proxy, next_idx = round_robin(proxies, 2)+ proxy, next_idx = cyclic_rotation(proxies, 2)assert proxy == "http://p3:8080"assert next_idx == 0 # Wraps around- def test_round_robin_wraps_index(self):- """Test that round_robin handles index overflow"""+ def test_cyclic_rotation_wraps_index(self):+ """Test that cyclic_rotation handles index overflow"""proxies = ["http://p1:8080", "http://p2:8080"]# Index larger than list length should wrap- proxy, next_idx = round_robin(proxies, 5)+ proxy, next_idx = cyclic_rotation(proxies, 5)assert proxy == "http://p2:8080" # 5 % 2 = 1assert next_idx == 0@@ -88,7 +88,7 @@ class TestProxyRotatorCreation:def test_non_callable_strategy_raises_error(self):"""Test that non-callable strategy raises TypeError"""with pytest.raises(TypeError, match="strategy must be callable"):- ProxyRotator(["http://p1:8080"], strategy="round_robin")+ ProxyRotator(["http://p1:8080"], strategy="cyclic_rotation")with pytest.raises(TypeError, match="strategy must be callable"):ProxyRotator(["http://p1:8080"], strategy=123)@@ -97,7 +97,7 @@ class TestProxyRotatorCreation:class TestProxyRotatorRotation:"""Test ProxyRotator rotation behavior"""- def test_get_proxy_round_robin(self):+ def test_get_proxy_cyclic_rotation(self):"""Test that get_proxy cycles through proxies in order"""proxies = ["http://p1:8080", "http://p2:8080", "http://p3:8080"]rotator = ProxyRotator(proxies)