🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: keep in-flight requests in checkpoints so resume does not lose them
yetvald1082bd2d8fc
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 6db6866..3972ff8 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -243,6 +243,7 @@ class CrawlerEngine:try:await self._process_request(request)finally:+ self.scheduler.complete(request)self._active_tasks -= 1def request_pause(self) -> None:diff --git a/scrapling/spiders/scheduler.py b/scrapling/spiders/scheduler.pyindex aca3277..331383d 100644--- a/scrapling/spiders/scheduler.py+++ b/scrapling/spiders/scheduler.py@@ -21,7 +21,7 @@ class Scheduler:self._queue: asyncio.PriorityQueue[tuple[int, int, Request]] = asyncio.PriorityQueue()self._seen: set[bytes] = set()self._counter = count()- # Mirror dict for snapshot without draining queue+ # Tracks every enqueued, not-yet-completed request (keyed by id) for snapshotsself._pending: dict[int, tuple[int, int, Request]] = {}self._include_kwargs = include_kwargsself._include_headers = include_headers@@ -40,16 +40,19 @@ class Scheduler:# Negative priority so higher priority = dequeued firstcounter = next(self._counter)item = (-request.priority, counter, request)- self._pending[counter] = item+ self._pending[id(request)] = itemawait self._queue.put(item)return Trueasync def dequeue(self) -> Request:- """Get the next request to process."""- _, counter, request = await self._queue.get()- self._pending.pop(counter, None)+ """Get the next request to process (stays tracked until complete())."""+ _, _, request = await self._queue.get()return request+ def complete(self, request: Request) -> None:+ """Mark a request as finished so it stops being tracked for checkpoints."""+ self._pending.pop(id(request), None)+def __len__(self) -> int:return self._queue.qsize()@@ -74,7 +77,7 @@ class Scheduler:for request in data.requests:counter = next(self._counter)item = (-request.priority, counter, request)- self._pending[counter] = item+ self._pending[id(request)] = itemself._queue.put_nowait(item)log.info(f"Scheduler restored: {len(data.requests)} requests, {len(data.seen)} seen")diff --git a/tests/spiders/test_scheduler.py b/tests/spiders/test_scheduler.pyindex a7365aa..8744e1c 100644--- a/tests/spiders/test_scheduler.py+++ b/tests/spiders/test_scheduler.py@@ -228,22 +228,26 @@ class TestSchedulerSnapshot:assert b"new_fingerprint_bytes" not in original_seen@pytest.mark.asyncio- async def test_snapshot_excludes_dequeued_requests(self):- """Test snapshot only includes pending requests."""+ async def test_snapshot_includes_in_flight_requests(self):+ """Test snapshot retains dequeued-but-not-completed (in-flight) requests."""scheduler = Scheduler()await scheduler.enqueue(Request("https://example.com/1"))await scheduler.enqueue(Request("https://example.com/2"))await scheduler.enqueue(Request("https://example.com/3"))- # Dequeue one- await scheduler.dequeue()+ in_flight = await scheduler.dequeue()requests, seen = scheduler.snapshot()- # Snapshot should only have 2 pending requests+ assert len(requests) == 3+ assert in_flight.url in {r.url for r in requests}+ assert len(seen) == 3++ scheduler.complete(in_flight)+ requests, seen = scheduler.snapshot()assert len(requests) == 2- # But seen should still have all 3 (deduplication tracking)+ assert in_flight.url not in {r.url for r in requests}assert len(seen) == 3@@ -356,9 +360,9 @@ class TestSchedulerIntegration:for i in range(5):await scheduler.enqueue(Request(f"https://example.com/{i}"))- # Process 2- await scheduler.dequeue()- await scheduler.dequeue()+ # Process 2 (mark them completed so they leave the snapshot)+ scheduler.complete(await scheduler.dequeue())+ scheduler.complete(await scheduler.dequeue())# Snapshot should show 3 pending, 5 seenrequests, seen = scheduler.snapshot()@@ -384,3 +388,58 @@ class TestSchedulerIntegration:result = await new_scheduler.enqueue(Request("https://example.com", sid="s1"))assert result is False # Duplicate filtered based on restored seen set+++class TestSchedulerInFlight:+ """Test in-flight tracking so checkpoints don't lose dequeued requests."""++ @pytest.mark.asyncio+ async def test_dequeue_keeps_request_tracked(self):+ """Dequeue does not drop the request; it stays tracked until completed."""+ scheduler = Scheduler()++ await scheduler.enqueue(Request("https://example.com/1"))+ await scheduler.enqueue(Request("https://example.com/2"))++ request = await scheduler.dequeue()++ assert len(scheduler._pending) == 2+ assert id(request) in scheduler._pending++ @pytest.mark.asyncio+ async def test_complete_stops_tracking_request(self):+ """complete() drops the request from tracking (idempotently)."""+ scheduler = Scheduler()++ await scheduler.enqueue(Request("https://example.com/1"))+ request = await scheduler.dequeue()++ scheduler.complete(request)+ assert len(scheduler._pending) == 0++ scheduler.complete(request)+ scheduler.complete(Request("https://example.com/unknown"))+ assert len(scheduler._pending) == 0++ @pytest.mark.asyncio+ async def test_in_flight_request_survives_checkpoint_resume(self):+ """Regression (F1): a request in-flight at checkpoint time survives resume."""+ scheduler = Scheduler()++ await scheduler.enqueue(Request("https://example.com/A"))+ await scheduler.enqueue(Request("https://example.com/B"))++ a = await scheduler.dequeue()+ assert a.url == "https://example.com/A"++ requests, seen = scheduler.snapshot()+ data = CheckpointData(requests=requests, seen=seen)++ restored = Scheduler()+ restored.restore(data)++ fetchable = set()+ while not restored.is_empty:+ fetchable.add((await restored.dequeue()).url)++ assert fetchable == {"https://example.com/A", "https://example.com/B"}