Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: keep in-flight requests in checkpoints so resume does not lose them

yetvald1082bd2d8fc
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index 6db6866..3972ff8 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -243,6 +243,7 @@ class CrawlerEngine:
try:
await self._process_request(request)
finally:
+ self.scheduler.complete(request)
self._active_tasks -= 1
def request_pause(self) -> None:
diff --git a/scrapling/spiders/scheduler.py b/scrapling/spiders/scheduler.py
index aca3277..331383d 100644
--- a/scrapling/spiders/scheduler.py
+++ b/scrapling/spiders/scheduler.py
@@ -21,7 +21,7 @@ class Scheduler:
self._queue: asyncio.PriorityQueue[tuple[int, int, Request]] = asyncio.PriorityQueue()
self._seen: set[bytes] = set()
self._counter = count()
- # Mirror dict for snapshot without draining queue
+ # Tracks every enqueued, not-yet-completed request (keyed by id) for snapshots
self._pending: dict[int, tuple[int, int, Request]] = {}
self._include_kwargs = include_kwargs
self._include_headers = include_headers
@@ -40,16 +40,19 @@ class Scheduler:
# Negative priority so higher priority = dequeued first
counter = next(self._counter)
item = (-request.priority, counter, request)
- self._pending[counter] = item
+ self._pending[id(request)] = item
await self._queue.put(item)
return True
async def dequeue(self) -> Request:
- """Get the next request to process."""
- _, counter, request = await self._queue.get()
- self._pending.pop(counter, None)
+ """Get the next request to process (stays tracked until complete())."""
+ _, _, request = await self._queue.get()
return request
+ def complete(self, request: Request) -> None:
+ """Mark a request as finished so it stops being tracked for checkpoints."""
+ self._pending.pop(id(request), None)
+
def __len__(self) -> int:
return self._queue.qsize()
@@ -74,7 +77,7 @@ class Scheduler:
for request in data.requests:
counter = next(self._counter)
item = (-request.priority, counter, request)
- self._pending[counter] = item
+ self._pending[id(request)] = item
self._queue.put_nowait(item)
log.info(f"Scheduler restored: {len(data.requests)} requests, {len(data.seen)} seen")
diff --git a/tests/spiders/test_scheduler.py b/tests/spiders/test_scheduler.py
index a7365aa..8744e1c 100644
--- a/tests/spiders/test_scheduler.py
+++ b/tests/spiders/test_scheduler.py
@@ -228,22 +228,26 @@ class TestSchedulerSnapshot:
assert b"new_fingerprint_bytes" not in original_seen
@pytest.mark.asyncio
- async def test_snapshot_excludes_dequeued_requests(self):
- """Test snapshot only includes pending requests."""
+ async def test_snapshot_includes_in_flight_requests(self):
+ """Test snapshot retains dequeued-but-not-completed (in-flight) requests."""
scheduler = Scheduler()
await scheduler.enqueue(Request("https://example.com/1"))
await scheduler.enqueue(Request("https://example.com/2"))
await scheduler.enqueue(Request("https://example.com/3"))
- # Dequeue one
- await scheduler.dequeue()
+ in_flight = await scheduler.dequeue()
requests, seen = scheduler.snapshot()
- # Snapshot should only have 2 pending requests
+ assert len(requests) == 3
+ assert in_flight.url in {r.url for r in requests}
+ assert len(seen) == 3
+
+ scheduler.complete(in_flight)
+ requests, seen = scheduler.snapshot()
assert len(requests) == 2
- # But seen should still have all 3 (deduplication tracking)
+ assert in_flight.url not in {r.url for r in requests}
assert len(seen) == 3
@@ -356,9 +360,9 @@ class TestSchedulerIntegration:
for i in range(5):
await scheduler.enqueue(Request(f"https://example.com/{i}"))
- # Process 2
- await scheduler.dequeue()
- await scheduler.dequeue()
+ # Process 2 (mark them completed so they leave the snapshot)
+ scheduler.complete(await scheduler.dequeue())
+ scheduler.complete(await scheduler.dequeue())
# Snapshot should show 3 pending, 5 seen
requests, seen = scheduler.snapshot()
@@ -384,3 +388,58 @@ class TestSchedulerIntegration:
result = await new_scheduler.enqueue(Request("https://example.com", sid="s1"))
assert result is False # Duplicate filtered based on restored seen set
+
+
+class TestSchedulerInFlight:
+ """Test in-flight tracking so checkpoints don't lose dequeued requests."""
+
+ @pytest.mark.asyncio
+ async def test_dequeue_keeps_request_tracked(self):
+ """Dequeue does not drop the request; it stays tracked until completed."""
+ scheduler = Scheduler()
+
+ await scheduler.enqueue(Request("https://example.com/1"))
+ await scheduler.enqueue(Request("https://example.com/2"))
+
+ request = await scheduler.dequeue()
+
+ assert len(scheduler._pending) == 2
+ assert id(request) in scheduler._pending
+
+ @pytest.mark.asyncio
+ async def test_complete_stops_tracking_request(self):
+ """complete() drops the request from tracking (idempotently)."""
+ scheduler = Scheduler()
+
+ await scheduler.enqueue(Request("https://example.com/1"))
+ request = await scheduler.dequeue()
+
+ scheduler.complete(request)
+ assert len(scheduler._pending) == 0
+
+ scheduler.complete(request)
+ scheduler.complete(Request("https://example.com/unknown"))
+ assert len(scheduler._pending) == 0
+
+ @pytest.mark.asyncio
+ async def test_in_flight_request_survives_checkpoint_resume(self):
+ """Regression (F1): a request in-flight at checkpoint time survives resume."""
+ scheduler = Scheduler()
+
+ await scheduler.enqueue(Request("https://example.com/A"))
+ await scheduler.enqueue(Request("https://example.com/B"))
+
+ a = await scheduler.dequeue()
+ assert a.url == "https://example.com/A"
+
+ requests, seen = scheduler.snapshot()
+ data = CheckpointData(requests=requests, seen=seen)
+
+ restored = Scheduler()
+ restored.restore(data)
+
+ fetchable = set()
+ while not restored.is_empty:
+ fetchable.add((await restored.dequeue()).url)
+
+ assert fetchable == {"https://example.com/A", "https://example.com/B"}