🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): keep the checkpoint when a crawl ends with an exception (#486)
Suliman Abdulrazzaq15373320e671
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 73eb390..b9eade1 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -380,6 +380,7 @@ class CrawlerEngine:await self._prefetch_robots_txt()+ finished = Falsetry:if not resuming:async for request in self.spider.start_requests():@@ -436,10 +437,11 @@ class CrawlerEngine:self._active_tasks += 1tg.start_soon(self._task_wrapper, request)+ finished = Truefinally:await self.spider.on_close()- # Clean up checkpoint files on successful completion (not paused)- if not self.paused and self._checkpoint_system_enabled:+ # Clean up checkpoint files only on successful completion (not paused, not crashed)+ if finished and not self.paused and self._checkpoint_system_enabled:await self._checkpoint_manager.cleanup()if self._autothrottle:diff --git a/tests/spiders/test_force_stop_checkpoint.py b/tests/spiders/test_force_stop_checkpoint.pyindex 41b8d8f..0026148 100644--- a/tests/spiders/test_force_stop_checkpoint.py+++ b/tests/spiders/test_force_stop_checkpoint.py@@ -224,6 +224,27 @@ class TestForceStopCheckpointPreservation:assert not checkpoint_path.exists()assert engine.paused is False+ @pytest.mark.anyio+ async def test_crash_keeps_checkpoint(self):+ """A crawl that ends with an exception must leave the checkpoint on disk."""++ class CrashingSpider(SlowSpider):+ async def is_blocked(self, response):+ if response.url.endswith("/page/5"):+ raise RuntimeError("boom")+ return False++ with tempfile.TemporaryDirectory() as tmpdir:+ spider = CrashingSpider(num_urls=20)+ engine = _make_engine(spider, MockSession(delay=0.05), crawldir=tmpdir, interval=0.001)++ with pytest.raises(Exception):+ await engine.crawl()++ assert (Path(tmpdir) / "checkpoint.pkl").exists(), "Checkpoint was deleted after the crawl crashed"+ assert engine.paused is False+ assert spider.on_close_calls == 1+@pytest.mark.anyioasync def test_force_stop_without_checkpoint_system(self):"""Force-stop without crawldir should not crash."""