Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): keep the checkpoint when a crawl ends with an exception (#486)

Suliman Abdulrazzaq15373320e671
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index 73eb390..b9eade1 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -380,6 +380,7 @@ class CrawlerEngine:
await self._prefetch_robots_txt()
+ finished = False
try:
if not resuming:
async for request in self.spider.start_requests():
@@ -436,10 +437,11 @@ class CrawlerEngine:
self._active_tasks += 1
tg.start_soon(self._task_wrapper, request)
+ finished = True
finally:
await self.spider.on_close()
- # Clean up checkpoint files on successful completion (not paused)
- if not self.paused and self._checkpoint_system_enabled:
+ # Clean up checkpoint files only on successful completion (not paused, not crashed)
+ if finished and not self.paused and self._checkpoint_system_enabled:
await self._checkpoint_manager.cleanup()
if self._autothrottle:
diff --git a/tests/spiders/test_force_stop_checkpoint.py b/tests/spiders/test_force_stop_checkpoint.py
index 41b8d8f..0026148 100644
--- a/tests/spiders/test_force_stop_checkpoint.py
+++ b/tests/spiders/test_force_stop_checkpoint.py
@@ -224,6 +224,27 @@ class TestForceStopCheckpointPreservation:
assert not checkpoint_path.exists()
assert engine.paused is False
+ @pytest.mark.anyio
+ async def test_crash_keeps_checkpoint(self):
+ """A crawl that ends with an exception must leave the checkpoint on disk."""
+
+ class CrashingSpider(SlowSpider):
+ async def is_blocked(self, response):
+ if response.url.endswith("/page/5"):
+ raise RuntimeError("boom")
+ return False
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ spider = CrashingSpider(num_urls=20)
+ engine = _make_engine(spider, MockSession(delay=0.05), crawldir=tmpdir, interval=0.001)
+
+ with pytest.raises(Exception):
+ await engine.crawl()
+
+ assert (Path(tmpdir) / "checkpoint.pkl").exists(), "Checkpoint was deleted after the crawl crashed"
+ assert engine.paused is False
+ assert spider.on_close_calls == 1
+
@pytest.mark.anyio
async def test_force_stop_without_checkpoint_system(self):
"""Force-stop without crawldir should not crash."""