Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate ScrapyCommand.set_crawler(), remove/fix some dead code. (#7276)

Andrey Rakhmatullin9f4651151d70
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index d5945f6f5..d666ca796 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -7,13 +7,14 @@ from __future__ import annotations
import argparse
import builtins
import os
+import warnings
from abc import ABC, abstractmethod
from pathlib import Path
from typing import TYPE_CHECKING, Any
from twisted.python import failure
-from scrapy.exceptions import UsageError
+from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
if TYPE_CHECKING:
@@ -36,7 +37,12 @@ class ScrapyCommand(ABC):
def __init__(self) -> None:
self.settings: Settings | None = None # set in scrapy.cmdline
- def set_crawler(self, crawler: Crawler) -> None:
+ def set_crawler(self, crawler: Crawler) -> None: # pragma: no cover
+ warnings.warn(
+ "ScrapyCommand.set_crawler() is deprecated",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
if hasattr(self, "_crawler"):
raise RuntimeError("crawler already set")
self._crawler: Crawler = crawler
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 695f6ef3e..701f7cb46 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -66,7 +66,7 @@ class Slot:
self.queue: deque[QueueTuple] = deque()
self.active: set[Request] = set()
self.active_size: int = 0
- self.itemproc_size: int = 0
+ self.itemproc_size: int = 0 # just for scrapy.utils.engine.get_engine_status()
self.closing: Deferred[Spider] | None = None
def add_response_request(
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index 233bc62fe..8d146b496 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -96,23 +96,21 @@ class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):
pass
-class Https11WrongHostnameTestCase(
- HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase
-):
+class TestHttps11WrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase):
pass
-class Https11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
+class TestHttps11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
pass
-class Https11InvalidDNSPattern(
+class TestHttps11InvalidDNSPattern(
HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
):
pass
-class Https11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):
+class TestHttps11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):
pass
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index e94591038..548c0d8ee 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -89,20 +89,6 @@ class TestBase:
assert response1.headers == response2.headers
assert response1.body == response2.body
- def assertEqualRequest(self, request1, request2):
- assert request1.url == request2.url
- assert request1.headers == request2.headers
- assert request1.body == request2.body
-
- def assertEqualRequestButWithCacheValidators(self, request1, request2):
- assert request1.url == request2.url
- assert b"If-None-Match" not in request1.headers
- assert b"If-Modified-Since" not in request1.headers
- assert any(
- h in request2.headers for h in (b"If-None-Match", b"If-Modified-Since")
- )
- assert request1.body == request2.body
-
class StorageTestMixin:
"""Mixin containing storage-specific test methods."""
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 0c0432b3c..131beeead 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -26,6 +26,7 @@ from scrapy.http import Headers, Request, Response
from scrapy.item import Field, Item
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
+from scrapy.statscollectors import MemoryStatsCollector
from scrapy.utils.defer import (
_schedule_coro,
deferred_from_coro,
@@ -42,7 +43,6 @@ if TYPE_CHECKING:
from scrapy.core.scheduler import Scheduler
from scrapy.crawler import Crawler
- from scrapy.statscollectors import MemoryStatsCollector
from tests.mockserver.http import MockServer
@@ -741,7 +741,8 @@ class TestEngineCloseSpider:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
- del cast("MemoryStatsCollector", crawler.stats).spider_stats
+ assert isinstance(crawler.stats, MemoryStatsCollector)
+ del crawler.stats.spider_stats
await engine.close_spider_async()
assert "Stats close failure" in caplog.text
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index 2cc23c491..86517e17b 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -98,7 +98,7 @@ class TestPeriodicLog:
ext.spider_opened(spider)
ext.set_a()
a = ext.log_delta()
- ext.set_a()
+ ext.set_b()
b = ext.log_delta()
ext.spider_closed(spider, reason="finished")
return ext, a, b
@@ -164,7 +164,7 @@ class TestPeriodicLog:
ext.spider_opened(spider)
ext.set_a()
a = ext.log_crawler_stats()
- ext.set_a()
+ ext.set_b()
b = ext.log_crawler_stats()
ext.spider_closed(spider, reason="finished")
return ext, a, b
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index ab7341d29..ff9b0a3f9 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -288,7 +288,7 @@ class IsExportingListener:
if self.start_without_finish:
self.start_without_finish = False
else:
- self.finish_before_start = True
+ self.finish_without_start = True
class ExceptionJsonItemExporter(JsonItemExporter):
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 224158e7f..7cfc7ed26 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -55,17 +55,6 @@ class ProcessorItemLoader(NameItemLoader):
name_in = MapCompose(lambda v: v.title())
-class DefaultedItemLoader(NameItemLoader):
- default_input_processor = MapCompose(lambda v: v[:-1])
-
-
-# test processors
-def processor_with_args(value, other=None, loader_context=None):
- if "key" in loader_context:
- return loader_context["key"]
- return value
-
-
class TestBasicItemLoader:
def test_add_value_on_unknown_field(self):
il = ProcessorItemLoader()
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index bccfc4432..3fbe9c94e 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -320,6 +320,11 @@ class TestMain:
DeprecatedWrapSpider,
)
+ @coroutine_test
+ async def test_universal_mw_uses_process_start(self):
+ """Test that process_start_requests() isn't used when process_start() exists."""
+ await self._test([UniversalSpiderMiddleware], ModernWrapSpider, [ITEM_B])
+
async def _test_sleep(self, spider_middlewares):
class TestSpider(Spider):
name = "test"
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 296bd6224..1b062d28d 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -6,7 +6,7 @@ from asyncio import Future
from typing import TYPE_CHECKING, Any
import pytest
-from twisted.internet.defer import Deferred, inlineCallbacks, succeed
+from twisted.internet.defer import Deferred, inlineCallbacks
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import (
@@ -65,26 +65,6 @@ class TestMustbeDeferred:
yield dfd
-def cb1(value, arg1, arg2):
- return f"(cb1 {value} {arg1} {arg2})"
-
-
-def cb2(value, arg1, arg2):
- return succeed(f"(cb2 {value} {arg1} {arg2})")
-
-
-def cb3(value, arg1, arg2):
- return f"(cb3 {value} {arg1} {arg2})"
-
-
-def cb_fail(value, arg1, arg2):
- raise TypeError
-
-
-def eb1(failure, arg1, arg2):
- return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})"
-
-
class TestIterErrback:
def test_iter_errback_good(self):
def itergood() -> Generator[int, None, None]: