Scrapy, a fast high-level web crawling & scraping framework for Python.
Align peek and pop (#7837)
Adrian1a6c8c324d65
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex ba9a648c8..a01183d0f 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -196,6 +196,9 @@ class ScrapyPriorityQueue:q.close()self.changed = True+ # A recorded priority may have no queue to restore, e.g. if it only+ # ever held a request that failed to serialize, so curprio comes from+ # the queues that do exist and not from startprios.self._update_curprio()def qfactory(self, key: int) -> QueueProtocol:@@ -230,47 +233,40 @@ class ScrapyPriorityQueue:self.changed = Trueq = self.queues[priority]q.push(request) # this may fail (eg. serialization error)- if self.curprio is None or priority < self.curprio:+ # A queue class may drop a request instead of storing it, and a+ # priority with an empty queue must not become the current one, since+ # pop() and peek() expect to find a request there.+ if q and (self.curprio is None or priority < self.curprio):self.curprio = priority+ def _curqueue(self) -> QueueProtocol:+ assert self.curprio is not None+ # Whichever dict holds a non-empty queue at the current priority. The+ # other one may hold an empty queue left behind by a failed push, e.g.+ # on a serialization error, which would hide the next request.+ return self.queues.get(self.curprio) or self._start_queues[self.curprio]+def pop(self) -> Request | None:- while self.curprio is not None:- try:- q = self.queues[self.curprio]- except KeyError:- pass- else:- m = q.pop()- if not q:- del self.queues[self.curprio]- q.close()- self.changed = True- self._update_curprio()- return m- if self._start_queues:- try:- q = self._start_queues[self.curprio]- except KeyError:- self._update_curprio()- else:- m = q.pop()- if not q:- del self._start_queues[self.curprio]- q.close()- self.changed = True- self._update_curprio()- return m- else:- self._update_curprio()- return None+ if self.curprio is None:+ return None+ q = self._curqueue()+ request = q.pop()+ if not q:+ self._update_curprio()+ return requestdef _update_curprio(self) -> None:- prios = {- p- for queues in (self.queues, self._start_queues)- for p, q in queues.items()- if q- }+ # Keeping an empty queue would hold its storage open for nothing, and+ # make close() record a priority with nothing to restore from it.+ prios: set[int] = set()+ for queues in (self.queues, self._start_queues):+ for p, q in list(queues.items()):+ if q:+ prios.add(p)+ else:+ del queues[p]+ q.close()+ self.changed = Trueself.curprio = min(prios) if prios else Nonedef peek(self) -> Request | None:@@ -282,10 +278,9 @@ class ScrapyPriorityQueue:"""if self.curprio is None:return None- try:- queue = self._start_queues[self.curprio]- except KeyError:- queue = self.queues[self.curprio]+ # Picking the queue the same way as pop() is what makes the returned+ # request the one that pop() then returns.+ queue = self._curqueue()# Protocols can't declare optional membersreturn cast("Request", queue.peek()) # type: ignore[attr-defined]diff --git a/tests/test_pqueues.py b/tests/test_pqueues.pyindex eef4374ce..5fc879c53 100644--- a/tests/test_pqueues.py+++ b/tests/test_pqueues.py@@ -21,6 +21,15 @@ if TYPE_CHECKING:from scrapy.crawler import Crawler+class DroppingFifoMemoryQueue(FifoMemoryQueue): # type: ignore[valid-type,misc]+ """Queue that drops requests marked with ``drop`` in their metadata, the+ way a custom queue class could drop requests it does not want to store."""++ def push(self, request: Request) -> None:+ if not request.meta.get("drop"):+ self.q.append(request)++def _pop(queue: ScrapyPriorityQueue | DownloaderAwarePriorityQueue) -> Request:request = queue.pop()assert request is not None@@ -76,6 +85,169 @@ class TestPriorityQueue:assert _pop(queue).url == req3.urlassert not queue.close()+ def test_peek_after_draining_a_higher_priority_queue(+ self, crawler: Crawler, tmp_path: Path+ ) -> None:+ """Draining the queue of the current priority while start requests+ remain at a different one must not leave ``curprio`` pointing at a+ priority that no queue has, which would make ``peek()`` come up empty+ with requests still queued."""+ queue = ScrapyPriorityQueue.from_crawler(+ crawler,+ FifoMemoryQueue,+ str(tmp_path),+ start_queue_cls=FifoMemoryQueue,+ )+ start_request = Request(+ "https://example.org/start", meta={"is_start_request": True}+ )+ queue.push(start_request)+ # A redirect of a start request, which REDIRECT_PRIORITY_ADJUST puts at+ # a higher priority, i.e. in a separate, non-start queue.+ queue.push(Request("https://example.org/redirect", priority=2))++ assert _peek(queue).url == "https://example.org/redirect"+ assert _pop(queue).url == "https://example.org/redirect"+ assert _peek(queue).url == start_request.url+ assert _pop(queue).url == start_request.url+ assert queue.peek() is None+ queue.close()++ def test_peek_agrees_with_pop_on_start_requests(+ self, crawler: Crawler, tmp_path: Path+ ) -> None:+ """A start request and a non-start request at the same priority sit in+ separate queues, and ``peek()`` must report the one that ``pop()``+ returns, since a caller may peek to decide whether to pop."""+ queue = ScrapyPriorityQueue.from_crawler(+ crawler,+ FifoMemoryQueue,+ str(tmp_path),+ start_queue_cls=FifoMemoryQueue,+ )+ queue.push(+ Request("https://example.org/start", meta={"is_start_request": True})+ )+ queue.push(Request("https://example.org/other"))++ while len(queue):+ peeked = _peek(queue)+ assert _pop(queue).url == peeked.url+ queue.close()++ def test_peek_and_pop_skip_an_empty_queue_left_by_a_failed_push(+ self, crawler: Crawler, tmp_path: Path+ ) -> None:+ """A queue is created before the request is pushed into it, so a push+ that fails (e.g. a serialization error) leaves an empty queue behind at+ that priority. Neither ``peek()`` nor ``pop()`` may let it hide the+ request that the other dict holds at the same priority."""+ queue = ScrapyPriorityQueue.from_crawler(+ crawler,+ PickleFifoDiskQueue,+ str(tmp_path),+ start_queue_cls=PickleFifoDiskQueue,+ )+ with pytest.raises(ValueError, match="is not an instance method"):+ queue.push(+ Request(+ "https://example.org/lambda",+ callback=lambda response: None, # type: ignore[misc]+ )+ )+ assert queue.queues[0] is not None # the empty leftover+ assert len(queue) == 0++ start_request = Request(+ "https://example.org/start", meta={"is_start_request": True}+ )+ queue.push(start_request)+ assert len(queue) == 1+ assert _peek(queue).url == start_request.url+ assert _pop(queue).url == start_request.url+ assert len(queue) == 0+ assert queue.peek() is None+ assert queue.pop() is None+ queue.close()++ def test_empty_queues_are_dropped_on_refresh(+ self, crawler: Crawler, tmp_path: Path+ ) -> None:+ """The empty leftover of a failed push is forgotten (and closed) the+ next time the current priority is refreshed, rather than kept around+ holding its storage open."""+ queue = ScrapyPriorityQueue.from_crawler(+ crawler, PickleFifoDiskQueue, str(tmp_path)+ )+ with pytest.raises(ValueError, match="is not an instance method"):+ queue.push(+ Request(+ "https://example.org/lambda",+ callback=lambda response: None, # type: ignore[misc]+ )+ )+ assert set(queue.queues) == {0} # the empty leftover+ # A request at a different priority, whose queue emptying is what+ # triggers the refresh.+ queue.push(Request("https://example.org/1", priority=1))+ assert _pop(queue).url == "https://example.org/1"+ assert queue.queues == {}+ assert queue.curprio is None+ assert queue.pop() is None+ assert not queue.close()++ def test_push_into_a_queue_that_drops_the_request(+ self, crawler: Crawler, tmp_path: Path+ ) -> None:+ """A queue class may drop a request instead of storing it, and that+ priority must not become the current one, or ``peek()`` and ``pop()``+ would come up empty with requests still queued."""+ queue = ScrapyPriorityQueue.from_crawler(+ crawler, DroppingFifoMemoryQueue, str(tmp_path)+ )+ # A higher priority than the request below, so that it would become the+ # current priority if it were stored.+ queue.push(+ Request("https://example.org/dropped", priority=1, meta={"drop": True})+ )+ kept = Request("https://example.org/kept")+ queue.push(kept)++ assert len(queue) == 1+ assert _peek(queue).url == kept.url+ assert _pop(queue).url == kept.url+ assert queue.peek() is None+ assert queue.pop() is None+ assert not queue.close()++ def test_init_prios_without_a_restorable_queue(+ self, crawler: Crawler, tmp_path: Path+ ) -> None:+ """A priority recorded on close may have nothing to restore, e.g. if+ its only request failed to serialize. ``curprio`` must not point at it,+ or ``peek()`` raises :exc:`KeyError` on the resumed crawl."""+ queue = ScrapyPriorityQueue.from_crawler(+ crawler, PickleFifoDiskQueue, str(tmp_path)+ )+ with pytest.raises(ValueError, match="is not an instance method"):+ queue.push(+ Request(+ "https://example.org/lambda",+ callback=lambda response: None, # type: ignore[misc]+ )+ )+ startprios = queue.close()+ assert startprios == [0]++ queue2 = ScrapyPriorityQueue.from_crawler(+ crawler, PickleFifoDiskQueue, str(tmp_path), startprios+ )+ assert len(queue2) == 0+ assert queue2.curprio is None+ assert queue2.peek() is None+ assert queue2.pop() is None+ queue2.close()+def test_init_prios_with_start_queue(self, crawler: Crawler, tmp_path: Path) -> None: