Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

perf(session): improve page management and speed up execution

Abdullah7ba333c3c9f6
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index 8e94559..6445e00 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -44,16 +44,11 @@ class SyncSession:
) -> PageInfo: # pragma: no cover
"""Get a new page to use"""
- # Close all finished pages to ensure clean state
- self.page_pool.close_all_finished_pages()
-
# If we're at max capacity after cleanup, wait for busy pages to finish
if self.page_pool.pages_count >= self.max_pages:
start_time = time()
while time() - start_time < self._max_wait_for_page:
- # Wait for any pages to finish, then clean them up
sleep(0.05)
- self.page_pool.close_all_finished_pages()
if self.page_pool.pages_count < self.max_pages:
break
else:
@@ -105,16 +100,11 @@ class AsyncSession(SyncSession):
) -> PageInfo: # pragma: no cover
"""Get a new page to use"""
async with self._lock:
- # Close all finished pages to ensure clean state
- await self.page_pool.aclose_all_finished_pages()
-
# If we're at max capacity after cleanup, wait for busy pages to finish
if self.page_pool.pages_count >= self.max_pages:
start_time = time()
while time() - start_time < self._max_wait_for_page:
- # Wait for any pages to finish, then clean them up
await asyncio_sleep(0.05)
- await self.page_pool.aclose_all_finished_pages()
if self.page_pool.pages_count < self.max_pages:
break
else:
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index a4ca6f0..18ec633 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -381,8 +381,9 @@ class StealthySession(StealthySessionMixin, SyncSession):
page_info.page, first_response, final_response, params.selector_config
)
- # Mark the page as finished for next use
- page_info.mark_finished()
+ # Close the page, to free up resources
+ page_info.page.close()
+ self.page_pool.pages.remove(page_info)
return response
@@ -701,8 +702,9 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
page_info.page, first_response, final_response, params.selector_config
)
- # Mark the page as finished for next use
- page_info.mark_finished()
+ # Close the page, to free up resources
+ await page_info.page.close()
+ self.page_pool.pages.remove(page_info)
return response
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 70b3771..722d7c2 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -305,8 +305,9 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
page_info.page, first_response, final_response, params.selector_config
)
- # Mark the page as finished for next use
- page_info.mark_finished()
+ # Close the page, to free up resources
+ page_info.page.close()
+ self.page_pool.pages.remove(page_info)
return response
@@ -554,9 +555,9 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
page_info.page, first_response, final_response, params.selector_config
)
- # Mark the page as finished for next use
- page_info.mark_finished()
-
+ # Close the page, to free up resources
+ await page_info.page.close()
+ self.page_pool.pages.remove(page_info)
return response
except Exception as e: # pragma: no cover
diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py
index ffa4b22..8c80944 100644
--- a/scrapling/engines/_browsers/_page.py
+++ b/scrapling/engines/_browsers/_page.py
@@ -23,11 +23,6 @@ class PageInfo:
self.state = "busy"
self.url = url
- def mark_finished(self):
- """Mark the page as finished for new requests"""
- self.state = "finished"
- self.url = ""
-
def mark_error(self):
"""Mark the page as having an error"""
self.state = "error"
@@ -83,33 +78,3 @@ class PagePool:
"""Remove pages in error state"""
with self._lock:
self.pages = [p for p in self.pages if p.state != "error"]
-
- def close_all_finished_pages(self):
- """Close all pages in finished state and remove them from the pool"""
- with self._lock:
- pages_to_remove = []
- for page_info in self.pages:
- if page_info.state == "finished":
- try:
- page_info.page.close()
- except Exception:
- pass
- pages_to_remove.append(page_info)
-
- for page_info in pages_to_remove:
- self.pages.remove(page_info)
-
- async def aclose_all_finished_pages(self):
- """Async version: Close all pages in finished state and remove them from the pool"""
- with self._lock:
- pages_to_remove = []
- for page_info in self.pages:
- if page_info.state == "finished":
- try:
- await page_info.page.close()
- except Exception:
- pass
- pages_to_remove.append(page_info)
-
- for page_info in pages_to_remove:
- self.pages.remove(page_info)
diff --git a/tests/fetchers/async/test_camoufox_session.py b/tests/fetchers/async/test_camoufox_session.py
index 2971488..af9b32b 100644
--- a/tests/fetchers/async/test_camoufox_session.py
+++ b/tests/fetchers/async/test_camoufox_session.py
@@ -54,16 +54,18 @@ class TestAsyncStealthySession:
"""Test page pool creation and reuse"""
async with AsyncStealthySession() as session:
# The first request creates a page
- _ = await session.fetch(urls["basic"])
- assert session.page_pool.pages_count == 1
+ response = await session.fetch(urls["basic"])
+ assert response.status == 200
+ assert session.page_pool.pages_count == 0
# The second request should reuse the page
- _ = await session.fetch(urls["html"])
- assert session.page_pool.pages_count == 1
+ response = await session.fetch(urls["html"])
+ assert response.status == 200
+ assert session.page_pool.pages_count == 0
# Check pool stats
stats = session.get_pool_stats()
- assert stats["total_pages"] == 1
+ assert stats["total_pages"] == 0
assert stats["max_pages"] == 1
async def test_stealthy_session_with_options(self, urls):
diff --git a/tests/fetchers/async/test_dynamic_session.py b/tests/fetchers/async/test_dynamic_session.py
index d7a4ea9..e6f0b5a 100644
--- a/tests/fetchers/async/test_dynamic_session.py
+++ b/tests/fetchers/async/test_dynamic_session.py
@@ -53,16 +53,18 @@ class TestAsyncDynamicSession:
"""Test page pool creation and reuse"""
async with AsyncDynamicSession() as session:
# The first request creates a page
- _ = await session.fetch(urls["basic"])
- assert session.page_pool.pages_count == 1
-
+ response = await session.fetch(urls["basic"])
+ assert response.status == 200
+ assert session.page_pool.pages_count == 0
+
# The second request should reuse the page
- _ = await session.fetch(urls["html"])
- assert session.page_pool.pages_count == 1
+ response = await session.fetch(urls["html"])
+ assert response.status == 200
+ assert session.page_pool.pages_count == 0
# Check pool stats
stats = session.get_pool_stats()
- assert stats["total_pages"] == 1
+ assert stats["total_pages"] == 0
assert stats["max_pages"] == 1
async def test_dynamic_session_with_options(self, urls):
diff --git a/tests/fetchers/test_pages.py b/tests/fetchers/test_pages.py
index e3ba3bc..69dc4ba 100644
--- a/tests/fetchers/test_pages.py
+++ b/tests/fetchers/test_pages.py
@@ -24,10 +24,6 @@ class TestPageInfo:
assert page_info.state == "busy"
assert page_info.url == "https://example.com"
- page_info.mark_finished()
- assert page_info.state == "finished"
- assert page_info.url == ""
-
page_info.mark_error()
assert page_info.state == "error"
@@ -88,21 +84,7 @@ class TestPagePool:
with pytest.raises(RuntimeError):
pool.add_page(Mock())
- def test_get_ready_page(self):
- """Test getting ready page"""
- pool = PagePool(max_pages=3)
- # Add pages
- page1 = pool.add_page(Mock())
- page2 = pool.add_page(Mock())
-
- # Mark them as finished
- page1.mark_finished()
- page2.mark_finished()
-
- # test
- pool.close_all_finished_pages()
- assert pool.pages_count == 0
def test_cleanup_error_pages(self):
"""Test cleaning up error pages"""