🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: proxy rotation page pool leak
yetval6c6aabeb73a8
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex eefb9f3..2d42b08 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -196,11 +196,14 @@ class SyncSession:context_options = self._build_context_with_proxy(proxy)context: BrowserContext = self.browser.new_context(**context_options)+ page_info = Nonetry:context = self._initialize_context(self._config, context)page_info = self._get_page(timeout, extra_headers, disable_resources, blocked_domains, context=context)yield page_infofinally:+ if page_info is not None and page_info in self.page_pool.pages:+ self.page_pool.pages.remove(page_info)context.close()else:# Standard mode: use PagePool with persistent context@@ -380,6 +383,7 @@ class AsyncSession:context_options = self._build_context_with_proxy(proxy)context: AsyncBrowserContext = await self.browser.new_context(**context_options)+ page_info = Nonetry:context = await self._initialize_context(self._config, context)page_info = await self._get_page(@@ -387,6 +391,8 @@ class AsyncSession:)yield page_infofinally:+ if page_info is not None and page_info in self.page_pool.pages:+ self.page_pool.pages.remove(page_info)await context.close()else:# Standard mode: use PagePool with persistent contextdiff --git a/tests/fetchers/test_pages.py b/tests/fetchers/test_pages.pyindex 69dc4ba..b726c74 100644--- a/tests/fetchers/test_pages.py+++ b/tests/fetchers/test_pages.py@@ -84,6 +84,15 @@ class TestPagePool:with pytest.raises(RuntimeError):pool.add_page(Mock())+ def test_proxy_rotation_pool_leak(self):+ pool = PagePool(max_pages=1)+ page_info = pool.add_page(Mock())+ assert pool.pages_count == 1+ pool.pages.remove(page_info)+ assert pool.pages_count == 0+ pool.add_page(Mock())+ assert pool.pages_count == 1+def test_cleanup_error_pages(self):