🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(sessions): Fixing a bug with max pages counter
Karim shoair568396536718
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex 00c5567..8e94559 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -31,7 +31,7 @@ class SyncSession:def __init__(self, max_pages: int = 1):self.max_pages = max_pagesself.page_pool = PagePool(max_pages)- self.__max_wait_for_page = 60+ self._max_wait_for_page = 60self.playwright: Optional[Playwright] = Noneself.context: Optional[BrowserContext] = Noneself._closed = False@@ -50,7 +50,7 @@ class SyncSession:# If we're at max capacity after cleanup, wait for busy pages to finishif self.page_pool.pages_count >= self.max_pages:start_time = time()- while time() - start_time < self.__max_wait_for_page:+ while time() - start_time < self._max_wait_for_page:# Wait for any pages to finish, then clean them upsleep(0.05)self.page_pool.close_all_finished_pages()@@ -58,7 +58,7 @@ class SyncSession:breakelse:raise TimeoutError(- f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period"+ f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period")page = self.context.new_page()@@ -111,7 +111,7 @@ class AsyncSession(SyncSession):# If we're at max capacity after cleanup, wait for busy pages to finishif self.page_pool.pages_count >= self.max_pages:start_time = time()- while time() - start_time < self.__max_wait_for_page:+ while time() - start_time < self._max_wait_for_page:# Wait for any pages to finish, then clean them upawait asyncio_sleep(0.05)await self.page_pool.aclose_all_finished_pages()@@ -119,7 +119,7 @@ class AsyncSession(SyncSession):breakelse:raise TimeoutError(- f"No pages finished to clear place in the pool within the {self.__max_wait_for_page}s timeout period"+ f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period")page = await self.context.new_page()