🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): handle errors with manual pause during stream mode
Karim shoair25850db92c19
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 1abeb76..463a290 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -207,6 +207,7 @@ class CrawlerEngine:self._items.clear()self.paused = Falseself._pause_requested = False+ self._force_stop = Falseself.stats = CrawlStats(start_time=anyio.current_time())# Check for existing checkpointdiff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.pyindex bde8f78..649f4d3 100644--- a/scrapling/spiders/spider.py+++ b/scrapling/spiders/spider.py@@ -211,7 +211,9 @@ class Spider(ABC):manager.add("default", FetcherSession())def pause(self):- """Pause the crawling process if checkpoint system is enabled."""+ """Pause the crawling process. Requires crawldir to be set for checkpoint system."""+ if not self.crawldir:+ raise RuntimeError("Cannot pause without crawldir - checkpoint system not enabled")if self._engine:self._engine.request_pause()else: