Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: track in-flight requests by queue counter so duplicate instances are not collapsed

yetval95687a5e37b2
diff --git a/scrapling/spiders/scheduler.py b/scrapling/spiders/scheduler.py
index 331383d..0484a68 100644
--- a/scrapling/spiders/scheduler.py
+++ b/scrapling/spiders/scheduler.py
@@ -21,8 +21,9 @@ class Scheduler:
self._queue: asyncio.PriorityQueue[tuple[int, int, Request]] = asyncio.PriorityQueue()
self._seen: set[bytes] = set()
self._counter = count()
- # Tracks every enqueued, not-yet-completed request (keyed by id) for snapshots
+ # Mirror dict for snapshot without draining queue
self._pending: dict[int, tuple[int, int, Request]] = {}
+ self._inflight: dict[int, list[int]] = {}
self._include_kwargs = include_kwargs
self._include_headers = include_headers
self._keep_fragments = keep_fragments
@@ -40,18 +41,25 @@ class Scheduler:
# Negative priority so higher priority = dequeued first
counter = next(self._counter)
item = (-request.priority, counter, request)
- self._pending[id(request)] = item
+ self._pending[counter] = item
await self._queue.put(item)
return True
async def dequeue(self) -> Request:
"""Get the next request to process (stays tracked until complete())."""
- _, _, request = await self._queue.get()
+ _, counter, request = await self._queue.get()
+ self._inflight.setdefault(id(request), []).append(counter)
return request
def complete(self, request: Request) -> None:
"""Mark a request as finished so it stops being tracked for checkpoints."""
- self._pending.pop(id(request), None)
+ counters = self._inflight.get(id(request))
+ if not counters:
+ return
+ counter = counters.pop()
+ if not counters:
+ del self._inflight[id(request)]
+ self._pending.pop(counter, None)
def __len__(self) -> int:
return self._queue.qsize()
@@ -77,7 +85,7 @@ class Scheduler:
for request in data.requests:
counter = next(self._counter)
item = (-request.priority, counter, request)
- self._pending[id(request)] = item
+ self._pending[counter] = item
self._queue.put_nowait(item)
log.info(f"Scheduler restored: {len(data.requests)} requests, {len(data.seen)} seen")
diff --git a/tests/spiders/test_scheduler.py b/tests/spiders/test_scheduler.py
index 8744e1c..3646f59 100644
--- a/tests/spiders/test_scheduler.py
+++ b/tests/spiders/test_scheduler.py
@@ -360,7 +360,7 @@ class TestSchedulerIntegration:
for i in range(5):
await scheduler.enqueue(Request(f"https://example.com/{i}"))
- # Process 2 (mark them completed so they leave the snapshot)
+ # Process 2
scheduler.complete(await scheduler.dequeue())
scheduler.complete(await scheduler.dequeue())
@@ -403,8 +403,9 @@ class TestSchedulerInFlight:
request = await scheduler.dequeue()
- assert len(scheduler._pending) == 2
- assert id(request) in scheduler._pending
+ requests, _ = scheduler.snapshot()
+ assert len(requests) == 2
+ assert request.url in {r.url for r in requests}
@pytest.mark.asyncio
async def test_complete_stops_tracking_request(self):
@@ -415,11 +416,30 @@ class TestSchedulerInFlight:
request = await scheduler.dequeue()
scheduler.complete(request)
- assert len(scheduler._pending) == 0
+ requests, _ = scheduler.snapshot()
+ assert requests == []
scheduler.complete(request)
scheduler.complete(Request("https://example.com/unknown"))
- assert len(scheduler._pending) == 0
+ requests, _ = scheduler.snapshot()
+ assert requests == []
+
+ @pytest.mark.asyncio
+ async def test_duplicate_instance_tracked_per_enqueue(self):
+ """Re-enqueuing the same Request instance (dont_filter) tracks each copy."""
+ scheduler = Scheduler()
+ request = Request("https://example.com/retry", dont_filter=True)
+
+ await scheduler.enqueue(request)
+ await scheduler.enqueue(request)
+
+ assert len(scheduler) == 2
+ requests, _ = scheduler.snapshot()
+ assert len(requests) == 2
+
+ scheduler.complete(await scheduler.dequeue())
+ requests, _ = scheduler.snapshot()
+ assert len(requests) == 1
@pytest.mark.asyncio
async def test_in_flight_request_survives_checkpoint_resume(self):