🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: track in-flight requests by queue counter so duplicate instances are not collapsed
yetval95687a5e37b2
diff --git a/scrapling/spiders/scheduler.py b/scrapling/spiders/scheduler.pyindex 331383d..0484a68 100644--- a/scrapling/spiders/scheduler.py+++ b/scrapling/spiders/scheduler.py@@ -21,8 +21,9 @@ class Scheduler:self._queue: asyncio.PriorityQueue[tuple[int, int, Request]] = asyncio.PriorityQueue()self._seen: set[bytes] = set()self._counter = count()- # Tracks every enqueued, not-yet-completed request (keyed by id) for snapshots+ # Mirror dict for snapshot without draining queueself._pending: dict[int, tuple[int, int, Request]] = {}+ self._inflight: dict[int, list[int]] = {}self._include_kwargs = include_kwargsself._include_headers = include_headersself._keep_fragments = keep_fragments@@ -40,18 +41,25 @@ class Scheduler:# Negative priority so higher priority = dequeued firstcounter = next(self._counter)item = (-request.priority, counter, request)- self._pending[id(request)] = item+ self._pending[counter] = itemawait self._queue.put(item)return Trueasync def dequeue(self) -> Request:"""Get the next request to process (stays tracked until complete())."""- _, _, request = await self._queue.get()+ _, counter, request = await self._queue.get()+ self._inflight.setdefault(id(request), []).append(counter)return requestdef complete(self, request: Request) -> None:"""Mark a request as finished so it stops being tracked for checkpoints."""- self._pending.pop(id(request), None)+ counters = self._inflight.get(id(request))+ if not counters:+ return+ counter = counters.pop()+ if not counters:+ del self._inflight[id(request)]+ self._pending.pop(counter, None)def __len__(self) -> int:return self._queue.qsize()@@ -77,7 +85,7 @@ class Scheduler:for request in data.requests:counter = next(self._counter)item = (-request.priority, counter, request)- self._pending[id(request)] = item+ self._pending[counter] = itemself._queue.put_nowait(item)log.info(f"Scheduler restored: {len(data.requests)} requests, {len(data.seen)} seen")diff --git a/tests/spiders/test_scheduler.py b/tests/spiders/test_scheduler.pyindex 8744e1c..3646f59 100644--- a/tests/spiders/test_scheduler.py+++ b/tests/spiders/test_scheduler.py@@ -360,7 +360,7 @@ class TestSchedulerIntegration:for i in range(5):await scheduler.enqueue(Request(f"https://example.com/{i}"))- # Process 2 (mark them completed so they leave the snapshot)+ # Process 2scheduler.complete(await scheduler.dequeue())scheduler.complete(await scheduler.dequeue())@@ -403,8 +403,9 @@ class TestSchedulerInFlight:request = await scheduler.dequeue()- assert len(scheduler._pending) == 2- assert id(request) in scheduler._pending+ requests, _ = scheduler.snapshot()+ assert len(requests) == 2+ assert request.url in {r.url for r in requests}@pytest.mark.asyncioasync def test_complete_stops_tracking_request(self):@@ -415,11 +416,30 @@ class TestSchedulerInFlight:request = await scheduler.dequeue()scheduler.complete(request)- assert len(scheduler._pending) == 0+ requests, _ = scheduler.snapshot()+ assert requests == []scheduler.complete(request)scheduler.complete(Request("https://example.com/unknown"))- assert len(scheduler._pending) == 0+ requests, _ = scheduler.snapshot()+ assert requests == []++ @pytest.mark.asyncio+ async def test_duplicate_instance_tracked_per_enqueue(self):+ """Re-enqueuing the same Request instance (dont_filter) tracks each copy."""+ scheduler = Scheduler()+ request = Request("https://example.com/retry", dont_filter=True)++ await scheduler.enqueue(request)+ await scheduler.enqueue(request)++ assert len(scheduler) == 2+ requests, _ = scheduler.snapshot()+ assert len(requests) == 2++ scheduler.complete(await scheduler.dequeue())+ requests, _ = scheduler.snapshot()+ assert len(requests) == 1@pytest.mark.asyncioasync def test_in_flight_request_survives_checkpoint_resume(self):