Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate ScrapyCommand.set_crawler(), remove/fix some dead code. (#7276)
Andrey Rakhmatullin9f4651151d70
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex d5945f6f5..d666ca796 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -7,13 +7,14 @@ from __future__ import annotationsimport argparseimport builtinsimport os+import warningsfrom abc import ABC, abstractmethodfrom pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom twisted.python import failure-from scrapy.exceptions import UsageError+from scrapy.exceptions import ScrapyDeprecationWarning, UsageErrorfrom scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cliif TYPE_CHECKING:@@ -36,7 +37,12 @@ class ScrapyCommand(ABC):def __init__(self) -> None:self.settings: Settings | None = None # set in scrapy.cmdline- def set_crawler(self, crawler: Crawler) -> None:+ def set_crawler(self, crawler: Crawler) -> None: # pragma: no cover+ warnings.warn(+ "ScrapyCommand.set_crawler() is deprecated",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )if hasattr(self, "_crawler"):raise RuntimeError("crawler already set")self._crawler: Crawler = crawlerdiff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 695f6ef3e..701f7cb46 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -66,7 +66,7 @@ class Slot:self.queue: deque[QueueTuple] = deque()self.active: set[Request] = set()self.active_size: int = 0- self.itemproc_size: int = 0+ self.itemproc_size: int = 0 # just for scrapy.utils.engine.get_engine_status()self.closing: Deferred[Spider] | None = Nonedef add_response_request(diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex 233bc62fe..8d146b496 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -96,23 +96,21 @@ class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):pass-class Https11WrongHostnameTestCase(- HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase-):+class TestHttps11WrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase):pass-class Https11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):+class TestHttps11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):pass-class Https11InvalidDNSPattern(+class TestHttps11InvalidDNSPattern(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase):pass-class Https11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):+class TestHttps11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):passdiff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex e94591038..548c0d8ee 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -89,20 +89,6 @@ class TestBase:assert response1.headers == response2.headersassert response1.body == response2.body- def assertEqualRequest(self, request1, request2):- assert request1.url == request2.url- assert request1.headers == request2.headers- assert request1.body == request2.body-- def assertEqualRequestButWithCacheValidators(self, request1, request2):- assert request1.url == request2.url- assert b"If-None-Match" not in request1.headers- assert b"If-Modified-Since" not in request1.headers- assert any(- h in request2.headers for h in (b"If-None-Match", b"If-Modified-Since")- )- assert request1.body == request2.body-class StorageTestMixin:"""Mixin containing storage-specific test methods."""diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 0c0432b3c..131beeead 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -26,6 +26,7 @@ from scrapy.http import Headers, Request, Responsefrom scrapy.item import Field, Itemfrom scrapy.linkextractors import LinkExtractorfrom scrapy.spiders import Spider+from scrapy.statscollectors import MemoryStatsCollectorfrom scrapy.utils.defer import (_schedule_coro,deferred_from_coro,@@ -42,7 +43,6 @@ if TYPE_CHECKING:from scrapy.core.scheduler import Schedulerfrom scrapy.crawler import Crawler- from scrapy.statscollectors import MemoryStatsCollectorfrom tests.mockserver.http import MockServer@@ -741,7 +741,8 @@ class TestEngineCloseSpider:engine = ExecutionEngine(crawler, lambda _: None)crawler.engine = engineawait engine.open_spider_async()- del cast("MemoryStatsCollector", crawler.stats).spider_stats+ assert isinstance(crawler.stats, MemoryStatsCollector)+ del crawler.stats.spider_statsawait engine.close_spider_async()assert "Stats close failure" in caplog.textdiff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.pyindex 2cc23c491..86517e17b 100644--- a/tests/test_extension_periodic_log.py+++ b/tests/test_extension_periodic_log.py@@ -98,7 +98,7 @@ class TestPeriodicLog:ext.spider_opened(spider)ext.set_a()a = ext.log_delta()- ext.set_a()+ ext.set_b()b = ext.log_delta()ext.spider_closed(spider, reason="finished")return ext, a, b@@ -164,7 +164,7 @@ class TestPeriodicLog:ext.spider_opened(spider)ext.set_a()a = ext.log_crawler_stats()- ext.set_a()+ ext.set_b()b = ext.log_crawler_stats()ext.spider_closed(spider, reason="finished")return ext, a, bdiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex ab7341d29..ff9b0a3f9 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -288,7 +288,7 @@ class IsExportingListener:if self.start_without_finish:self.start_without_finish = Falseelse:- self.finish_before_start = True+ self.finish_without_start = Trueclass ExceptionJsonItemExporter(JsonItemExporter):diff --git a/tests/test_loader.py b/tests/test_loader.pyindex 224158e7f..7cfc7ed26 100644--- a/tests/test_loader.py+++ b/tests/test_loader.py@@ -55,17 +55,6 @@ class ProcessorItemLoader(NameItemLoader):name_in = MapCompose(lambda v: v.title())-class DefaultedItemLoader(NameItemLoader):- default_input_processor = MapCompose(lambda v: v[:-1])---# test processors-def processor_with_args(value, other=None, loader_context=None):- if "key" in loader_context:- return loader_context["key"]- return value--class TestBasicItemLoader:def test_add_value_on_unknown_field(self):il = ProcessorItemLoader()diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.pyindex bccfc4432..3fbe9c94e 100644--- a/tests/test_spidermiddleware_process_start.py+++ b/tests/test_spidermiddleware_process_start.py@@ -320,6 +320,11 @@ class TestMain:DeprecatedWrapSpider,)+ @coroutine_test+ async def test_universal_mw_uses_process_start(self):+ """Test that process_start_requests() isn't used when process_start() exists."""+ await self._test([UniversalSpiderMiddleware], ModernWrapSpider, [ITEM_B])+async def _test_sleep(self, spider_middlewares):class TestSpider(Spider):name = "test"diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 296bd6224..1b062d28d 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -6,7 +6,7 @@ from asyncio import Futurefrom typing import TYPE_CHECKING, Anyimport pytest-from twisted.internet.defer import Deferred, inlineCallbacks, succeed+from twisted.internet.defer import Deferred, inlineCallbacksfrom scrapy.utils.asyncgen import as_async_generator, collect_asyncgenfrom scrapy.utils.defer import (@@ -65,26 +65,6 @@ class TestMustbeDeferred:yield dfd-def cb1(value, arg1, arg2):- return f"(cb1 {value} {arg1} {arg2})"---def cb2(value, arg1, arg2):- return succeed(f"(cb2 {value} {arg1} {arg2})")---def cb3(value, arg1, arg2):- return f"(cb3 {value} {arg1} {arg2})"---def cb_fail(value, arg1, arg2):- raise TypeError---def eb1(failure, arg1, arg2):- return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})"--class TestIterErrback:def test_iter_errback_good(self):def itergood() -> Generator[int, None, None]: