Scrapy, a fast high-level web crawling & scraping framework for Python.
2.13.1 (#6832)
Adrián Chavesf98ffc71d25a
diff --git a/docs/news.rst b/docs/news.rstindex cf1c35893..ef3b549e7 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -3,6 +3,37 @@Release notes=============+.. _release-2.13.1:++Scrapy 2.13.1 (2025-05-28)+--------------------------++- Give callback requests precedence over start requests when priority values+ are the same.++ This makes changes from 2.13.0 to start request handling more intuitive and+ backward compatible. For scenarios where all requests have the same+ priorities, in 2.13.0 all start requests were sent before the first+ callback request. In 2.13.1, same as in 2.12 and lower, start requests are+ only sent when there are not enough pending callback requests to reach+ concurrency limits.++ (:issue:`6828`)++- Added a deepwiki_ badge to the README. (:issue:`6793`)++ .. _deepwiki: https://deepwiki.com/scrapy/scrapy++- Fixed a typo in the code example of :ref:`start-requests-lazy`.+ (:issue:`6812`, :issue:`6815`)++- Fixed a typo in the :ref:`coroutine-support` section of the documentation.+ (:issue:`6822`)++- Made this page more prominently listed in PyPI project links.+ (:issue:`6826`)++.. _release-2.13.0:Scrapy 2.13.0 (2025-05-08)diff --git a/pyproject.toml b/pyproject.tomlindex 47707e061..68c1e07bb 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -115,11 +115,17 @@ module = "twisted"implicit_reexport = true[tool.bumpversion]-current_version = "2.13.0"+current_version = "2.13.1"commit = truetag = truetag_name = "{new_version}"+[[tool.bumpversion.files]]+filename = "docs/news.rst"+search = "\\(unreleased\\)$"+replace = "({now:%Y-%m-%d})"+regex = true+[[tool.bumpversion.files]]filename = "scrapy/VERSION"diff --git a/scrapy/VERSION b/scrapy/VERSIONindex fb2c0766b..94f15e9cc 100644--- a/scrapy/VERSION+++ b/scrapy/VERSION@@ -1 +1 @@-2.13.0+2.13.1diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex 57d27b7cf..9ac447289 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -173,8 +173,8 @@ class Scheduler(BaseScheduler)::ref:`Start requests <start-requests>` are sent in the order they areyielded from :meth:`~scrapy.Spider.start`, and given the same- :attr:`~scrapy.http.Request.priority`, start requests take precedence over- other requests.+ :attr:`~scrapy.http.Request.priority`, other requests take precedence over+ start requests.You can set :setting:`SCHEDULER_START_MEMORY_QUEUE` and:setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` to handle start requestsdiff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex e6c6b8bf1..34b235d83 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -160,28 +160,32 @@ class ScrapyPriorityQueue:def pop(self) -> Request | None:while self.curprio is not None:+ try:+ q = self.queues[self.curprio]+ except KeyError:+ pass+ else:+ m = q.pop()+ if not q:+ del self.queues[self.curprio]+ q.close()+ if not self._start_queues:+ self._update_curprio()+ return mif self._start_queues:try:q = self._start_queues[self.curprio]except KeyError:- pass+ self._update_curprio()else:m = q.pop()if not q:del self._start_queues[self.curprio]q.close()+ self._update_curprio()return m- try:- q = self.queues[self.curprio]- except KeyError:- self._update_curprio()else:- m = q.pop()- if not q:- del self.queues[self.curprio]- q.close()- self._update_curprio()- return m+ self._update_curprio()return Nonedef _update_curprio(self) -> None:diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.pyindex 90af10f0e..c7dbc82d4 100644--- a/tests/test_engine_loop.py+++ b/tests/test_engine_loop.py@@ -189,9 +189,9 @@ class RequestSendOrderTestCase(TestCase):@deferred_f_from_coro_fasync def test_default(self):- """By default, start requests take priority over callback requests and+ """By default, callback requests take priority over start requests andare sent in order. Priority matters, but given the same priority, a- start request takes precedence."""+ callback request takes precedence."""nums = [1, 2, 3, 4, 5, 6]response_seconds = 0download_slots = 1@@ -207,13 +207,13 @@ class RequestSendOrderTestCase(TestCase):yield _request(1)for request in (- _request(4, priority=1),- _request(6),+ _request(2, priority=1),+ _request(5),):spider.crawler.engine._slot.scheduler.enqueue_request(request)- yield _request(5)- yield _request(2, priority=1)+ yield _request(6)yield _request(3, priority=1)+ yield _request(4, priority=1)def parse(spider, response):return@@ -249,13 +249,13 @@ class RequestSendOrderTestCase(TestCase):yield _request(1)for request in (- _request(4, priority=1),- _request(6),+ _request(2, priority=1),+ _request(5),):spider.crawler.engine._slot.scheduler.enqueue_request(request)- yield _request(5)+ yield _request(6)+ yield _request(4, priority=1)yield _request(3, priority=1)- yield _request(2, priority=1)def parse(spider, response):returndiff --git a/tests/test_pqueues.py b/tests/test_pqueues.pyindex d5c710ed2..b65f1b7e7 100644--- a/tests/test_pqueues.py+++ b/tests/test_pqueues.py@@ -7,6 +7,7 @@ from scrapy.http.request import Requestfrom scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueuefrom scrapy.spiders import Spiderfrom scrapy.squeues import FifoMemoryQueue+from scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.test import get_crawlerfrom tests.test_scheduler import MockDownloader, MockEngine@@ -155,3 +156,56 @@ class TestDownloaderAwarePriorityQueue:assert self.queue.peek().url == req3.urlassert self.queue.pop().url == req3.urlassert self.queue.peek() is None+++ ("input", "output"),+ [+ # By default, start requests are FIFO, other requests are LIFO.+ ([{}, {}], [2, 1]),+ ([{"start": True}, {"start": True}], [1, 2]),+ # Priority matters.+ ([{"priority": 1}, {"start": True}], [1, 2]),+ ([{}, {"start": True, "priority": 1}], [2, 1]),+ # For the same priority, start requests pop last.+ ([{}, {"start": True}], [1, 2]),+ ([{"start": True}, {}], [2, 1]),+ ],+)+def test_pop_order(input, output):+ def make_url(index):+ return f"https://toscrape.com/{index}"++ def make_request(index, data):+ meta = {}+ if data.get("start", False):+ meta["is_start_request"] = True+ return Request(+ url=make_url(index),+ priority=data.get("priority", 0),+ meta=meta,+ )++ input_requests = [+ make_request(index, data) for index, data in enumerate(input, start=1)+ ]+ expected_output_urls = [make_url(index) for index in output]++ crawler = get_crawler(Spider)+ settings = crawler.settings+ queue = build_from_crawler(+ ScrapyPriorityQueue,+ crawler,+ downstream_queue_cls=load_object(settings["SCHEDULER_MEMORY_QUEUE"]),+ key="",+ start_queue_cls=load_object(settings["SCHEDULER_START_MEMORY_QUEUE"]),+ )++ for request in input_requests:+ queue.push(request)++ actual_output_urls = []+ while request := queue.pop():+ actual_output_urls.append(request.url)++ assert actual_output_urls == expected_output_urls