Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(browsers): Close pages that give error

Karim shoaird5b9ed11b282
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index aef197a..3d39e9d 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -158,6 +158,8 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
except Exception as e:
page_info.mark_error()
+ page_info.page.close()
+ self.page_pool.pages.remove(page_info)
raise e
@@ -298,4 +300,6 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
except Exception as e: # pragma: no cover
page_info.mark_error()
+ await page_info.page.close()
+ self.page_pool.pages.remove(page_info)
raise e
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index 6883fdf..ade3ab9 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -250,6 +250,8 @@ class StealthySession(SyncSession, StealthySessionMixin):
except Exception as e:
page_info.mark_error()
+ page_info.page.close()
+ self.page_pool.pages.remove(page_info)
raise e
@@ -467,4 +469,6 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
except Exception as e: # pragma: no cover
page_info.mark_error()
+ await page_info.page.close()
+ self.page_pool.pages.remove(page_info)
raise e
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index e4210e9..dce7b08 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -1,5 +1,3 @@
-"""Crawler engine that orchestrates the crawling process."""
-
import json
import anyio