Scrapy, a fast high-level web crawling & scraping framework for Python.
Add Downloader.get_slot_key() without a spider parameter (#6352)
Sanchay Kumarc9ef5209365b
diff --git a/docs/news.rst b/docs/news.rstindex fafea0bf8..7db4e59a1 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -3,6 +3,20 @@Release notes=============++.. _release-VERSION:++Scrapy VERSION (YYYY-MM-DD)+---------------------------++Deprecations+~~~~~~~~~~~~++- :func:`scrapy.core.downloader.Downloader._get_slot_key` is now deprecated.+ Consider using its corresponding public method get_slot_key() instead.+ (:issue:`6340`)++.. _release-2.11.1:Scrapy 2.11.1 (2024-02-14)diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 98e1af6fb..0ab3bdb77 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -1,4 +1,5 @@import random+import warningsfrom collections import dequefrom datetime import datetimefrom time import time@@ -10,6 +11,7 @@ from twisted.internet.defer import Deferredfrom scrapy import Request, Spider, signalsfrom scrapy.core.downloader.handlers import DownloadHandlersfrom scrapy.core.downloader.middleware import DownloaderMiddlewareManager+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Responsefrom scrapy.resolver import dnscachefrom scrapy.settings import BaseSettings@@ -125,7 +127,7 @@ class Downloader:return len(self.active) >= self.total_concurrencydef _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]:- key = self._get_slot_key(request, spider)+ key = self.get_slot_key(request)if key not in self.slots:slot_settings = self.per_slot_settings.get(key, {})conc = (@@ -143,7 +145,7 @@ class Downloader:return key, self.slots[key]- def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:+ def get_slot_key(self, request: Request) -> str:if self.DOWNLOAD_SLOT in request.meta:return cast(str, request.meta[self.DOWNLOAD_SLOT])@@ -153,6 +155,14 @@ class Downloader:return key+ def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:+ warnings.warn(+ "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return self.get_slot_key(request)+def _enqueue_request(self, request: Request, spider: Spider) -> Deferred:key, slot = self._get_slot(request, spider)request.meta[self.DOWNLOAD_SLOT] = keydiff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex 773825c5e..58a47ef0f 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -180,7 +180,7 @@ class DownloaderInterface:return [(self._active_downloads(slot), slot) for slot in possible_slots]def get_slot_key(self, request: Request) -> str:- return self.downloader._get_slot_key(request, None)+ return self.downloader.get_slot_key(request)def _active_downloads(self, slot: str) -> int:"""Return a number of requests in a Downloader for a given slot"""diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex 37099dae6..02b50baa3 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -25,7 +25,7 @@ class MockDownloader:def __init__(self):self.slots = {}- def _get_slot_key(self, request, spider):+ def get_slot_key(self, request):if Downloader.DOWNLOAD_SLOT in request.meta:return request.meta[Downloader.DOWNLOAD_SLOT]@@ -273,14 +273,14 @@ class DownloaderAwareSchedulerTestMixin:while self.scheduler.has_pending_requests():request = self.scheduler.next_request()# pylint: disable=protected-access- slot = downloader._get_slot_key(request, None)+ slot = downloader.get_slot_key(request)dequeued_slots.append(slot)downloader.increment(slot)requests.append(request)for request in requests:# pylint: disable=protected-access- slot = downloader._get_slot_key(request, None)+ slot = downloader.get_slot_key(request)downloader.decrement(slot)self.assertTrue(