Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): handle errors with manual pause during stream mode

Karim shoair25850db92c19
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index 1abeb76..463a290 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -207,6 +207,7 @@ class CrawlerEngine:
self._items.clear()
self.paused = False
self._pause_requested = False
+ self._force_stop = False
self.stats = CrawlStats(start_time=anyio.current_time())
# Check for existing checkpoint
diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py
index bde8f78..649f4d3 100644
--- a/scrapling/spiders/spider.py
+++ b/scrapling/spiders/spider.py
@@ -211,7 +211,9 @@ class Spider(ABC):
manager.add("default", FetcherSession())
def pause(self):
- """Pause the crawling process if checkpoint system is enabled."""
+ """Pause the crawling process. Requires crawldir to be set for checkpoint system."""
+ if not self.crawldir:
+ raise RuntimeError("Cannot pause without crawldir - checkpoint system not enabled")
if self._engine:
self._engine.request_pause()
else: