Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: proxy rotation page pool leak

yetval6c6aabeb73a8
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index eefb9f3..2d42b08 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -196,11 +196,14 @@ class SyncSession:
context_options = self._build_context_with_proxy(proxy)
context: BrowserContext = self.browser.new_context(**context_options)
+ page_info = None
try:
context = self._initialize_context(self._config, context)
page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains, context=context)
yield page_info
finally:
+ if page_info is not None and page_info in self.page_pool.pages:
+ self.page_pool.pages.remove(page_info)
context.close()
else:
# Standard mode: use PagePool with persistent context
@@ -380,6 +383,7 @@ class AsyncSession:
context_options = self._build_context_with_proxy(proxy)
context: AsyncBrowserContext = await self.browser.new_context(**context_options)
+ page_info = None
try:
context = await self._initialize_context(self._config, context)
page_info = await self._get_page(
@@ -387,6 +391,8 @@ class AsyncSession:
)
yield page_info
finally:
+ if page_info is not None and page_info in self.page_pool.pages:
+ self.page_pool.pages.remove(page_info)
await context.close()
else:
# Standard mode: use PagePool with persistent context
diff --git a/tests/fetchers/test_pages.py b/tests/fetchers/test_pages.py
index 69dc4ba..b726c74 100644
--- a/tests/fetchers/test_pages.py
+++ b/tests/fetchers/test_pages.py
@@ -84,6 +84,15 @@ class TestPagePool:
with pytest.raises(RuntimeError):
pool.add_page(Mock())
+ def test_proxy_rotation_pool_leak(self):
+ pool = PagePool(max_pages=1)
+ page_info = pool.add_page(Mock())
+ assert pool.pages_count == 1
+ pool.pages.remove(page_info)
+ assert pool.pages_count == 0
+ pool.add_page(Mock())
+ assert pool.pages_count == 1
+
def test_cleanup_error_pages(self):