Scrapy, a fast high-level web crawling & scraping framework for Python.
Drop start_requests() support. (#7490)
Andrey Rakhmatullinf3868e11fb0e
diff --git a/extras/qpsclient.py b/extras/qpsclient.pyindex 269b27336..8e5001c1d 100644--- a/extras/qpsclient.py+++ b/extras/qpsclient.py@@ -35,10 +35,6 @@ class QPSSpider(Spider):self.download_delay = float(self.download_delay)async def start(self):- for item_or_request in self.start_requests():- yield item_or_request-- def start_requests(self):url = self.benchurlif self.latency is not None:url += f"?latency={self.latency}"diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 67342099d..790317357 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -33,7 +33,6 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_nif TYPE_CHECKING:from collections.abc import Generator- from scrapy.crawler import Crawlerfrom scrapy.settings import BaseSettings@@ -60,76 +59,12 @@ class SpiderMiddlewareManager(MiddlewareManager):settings.get_component_priority_dict_with_base("SPIDER_MIDDLEWARES"))- def __init__(self, *middlewares: Any, crawler: Crawler | None = None) -> None:- self._check_deprecated_process_start_requests_use(middlewares)- super().__init__(*middlewares, crawler=crawler)-- def _check_deprecated_process_start_requests_use(- self, middlewares: tuple[Any, ...]- ) -> None:- deprecated_middlewares = [- middleware- for middleware in middlewares- if hasattr(middleware, "process_start_requests")- and not hasattr(middleware, "process_start")- ]- modern_middlewares = [- middleware- for middleware in middlewares- if not hasattr(middleware, "process_start_requests")- and hasattr(middleware, "process_start")- ]- if deprecated_middlewares and modern_middlewares:- raise ValueError(- "You are trying to combine spider middlewares that only "- "define the deprecated process_start_requests() method () "- "with spider middlewares that only define the "- "process_start() method (). This is not possible. You must "- "either disable or make universal 1 of those 2 sets of "- "spider middlewares. Making a spider middleware universal "- "means having it define both methods. See the release notes "- "of Scrapy 2.13 for details: "- "https://docs.scrapy.org/en/2.13/news.html"- )-- self._use_start_requests = bool(deprecated_middlewares)- if self._use_start_requests:- deprecated_middleware_list = ", ".join(- global_object_name(middleware.__class__)- for middleware in deprecated_middlewares- )- warn(- f"The following enabled spider middlewares, directly or "- f"through their parent classes, define the deprecated "- f"process_start_requests() method: "- f"{deprecated_middleware_list}. process_start_requests() has "- f"been deprecated in favor of a new method, process_start(), "- f"to support asynchronous code execution. "- f"process_start_requests() will stop being called in a future "- f"version of Scrapy. If you use Scrapy 2.13 or higher "- f"only, replace process_start_requests() with "- f"process_start(); note that process_start() is a coroutine "- f"(async def). If you need to maintain compatibility with "- f"lower Scrapy versions, when defining "- f"process_start_requests() in a spider middleware class, "- f"define process_start() as well. See the release notes of "- f"Scrapy 2.13 for details: "- f"https://docs.scrapy.org/en/2.13/news.html",- ScrapyDeprecationWarning,- stacklevel=2,- )-def _add_middleware(self, mw: Any) -> None:if hasattr(mw, "process_spider_input"):self.methods["process_spider_input"].append(mw.process_spider_input)self._check_mw_method_spider_arg(mw.process_spider_input)- if self._use_start_requests:- if hasattr(mw, "process_start_requests"):- self.methods["process_start_requests"].appendleft(- mw.process_start_requests- )- elif hasattr(mw, "process_start"):+ if hasattr(mw, "process_start"):self.methods["process_start"].appendleft(mw.process_start)process_spider_output = self._get_async_method_pair(mw, "process_spider_output")@@ -451,88 +386,8 @@ class SpiderMiddlewareManager(MiddlewareManager):)warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)self._set_compat_spider(spider)- self._check_deprecated_start_requests_use()- if self._use_start_requests:- sync_start = iter(self._spider.start_requests())- sync_start = await self._process_chain(- "process_start_requests", sync_start, always_add_spider=True- )- start: AsyncIterator[Any] = as_async_generator(sync_start)- else:- start = self._spider.start()- start = await self._process_chain("process_start", start)- return start-- def _check_deprecated_start_requests_use(self) -> None:- start_requests_cls = None- start_cls = None- spidercls = self._spider.__class__- mro = spidercls.__mro__-- for cls in mro:- cls_dict = cls.__dict__- if start_requests_cls is None and "start_requests" in cls_dict:- start_requests_cls = cls- if start_cls is None and "start" in cls_dict:- start_cls = cls- if start_requests_cls is not None and start_cls is not None:- break-- # Spider defines both, start_requests and start.- assert start_requests_cls is not None- assert start_cls is not None-- if (- start_requests_cls is not Spider- and start_cls is not start_requests_cls- and mro.index(start_requests_cls) < mro.index(start_cls)- ):- src = global_object_name(start_requests_cls)- if start_requests_cls is not spidercls:- src += f" (inherited by {global_object_name(spidercls)})"- warn(- f"{src} defines the deprecated start_requests() method. "- f"start_requests() has been deprecated in favor of a new "- f"method, start(), to support asynchronous code "- f"execution. start_requests() will stop being called in a "- f"future version of Scrapy. If you use Scrapy 2.13 or "- f"higher only, replace start_requests() with start(); "- f"note that start() is a coroutine (async def). If you "- f"need to maintain compatibility with lower Scrapy versions, "- f"when overriding start_requests() in a spider class, "- f"override start() as well; you can use super() to "- f"reuse the inherited start() implementation without "- f"copy-pasting. See the release notes of Scrapy 2.13 for "- f"details: https://docs.scrapy.org/en/2.13/news.html",- ScrapyDeprecationWarning,- stacklevel=2,- )-- if (- self._use_start_requests- and start_cls is not Spider- and start_requests_cls is not start_cls- and mro.index(start_cls) < mro.index(start_requests_cls)- ):- src = global_object_name(start_cls)- if start_cls is not spidercls:- src += f" (inherited by {global_object_name(spidercls)})"- raise ValueError(- f"{src} does not define the deprecated start_requests() "- f"method. However, one or more of your enabled spider "- f"middlewares (reported in an earlier deprecation warning) "- f"define the process_start_requests() method, and not the "- f"process_start() method, making them only compatible with "- f"(deprecated) spiders that define the start_requests() "- f"method. To solve this issue, disable the offending spider "- f"middlewares, upgrade them as described in that earlier "- f"deprecation warning, or make your spider compatible with "- f"deprecated spider middlewares (and earlier Scrapy versions) "- f"by defining a sync start_requests() method that works "- f"similarly to its existing start() method. See the "- f"release notes of Scrapy 2.13 for details: "- f"https://docs.scrapy.org/en/2.13/news.html"- )+ start = self._spider.start()+ return await self._process_chain("process_start", start)# This method is only needed until _async compatibility methods are removed.@staticmethoddiff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.pyindex 889fc6df1..e09f2d10e 100644--- a/scrapy/spidermiddlewares/base.py+++ b/scrapy/spidermiddlewares/base.py@@ -41,13 +41,6 @@ class BaseSpiderMiddleware:def from_crawler(cls, crawler: Crawler) -> Self:return cls(crawler)- def process_start_requests(- self, start: Iterable[Any], spider: Spider- ) -> Iterable[Any]:- for o in start:- if (o := self._get_processed(o, None)) is not None:- yield o-async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:async for o in start:if (o := self._get_processed(o, None)) is not None:diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex 018e510c5..299a5d43f 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -7,17 +7,15 @@ See documentation in docs/topics/spiders.rstfrom __future__ import annotationsimport logging-import warningsfrom typing import TYPE_CHECKING, Any, castfrom scrapy import signals-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.utils.trackref import object_reffrom scrapy.utils.url import url_is_from_spiderif TYPE_CHECKING:- from collections.abc import AsyncIterator, Iterable+ from collections.abc import AsyncIteratorfrom twisted.internet.defer import Deferred@@ -127,24 +125,6 @@ class Spider(object_ref):.. seealso:: :ref:`start-requests`"""- with warnings.catch_warnings():- warnings.filterwarnings(- "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$"- )- for item_or_request in self.start_requests():- yield item_or_request-- def start_requests(self) -> Iterable[Any]:- warnings.warn(- (- "The Spider.start_requests() method is deprecated, use "- "Spider.start() instead. If you are calling "- "super().start_requests() from a Spider.start() override, "- "iterate super().start() instead."- ),- ScrapyDeprecationWarning,- stacklevel=2,- )if not self.start_urls and hasattr(self, "start_url"):raise AttributeError("Crawling could not start: 'start_urls' not found "diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex d87ffccaf..2a80b8d24 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -54,10 +54,6 @@ class SitemapSpider(Spider):self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]async def start(self) -> AsyncIterator[Any]:- for item_or_request in self.start_requests():- yield item_or_request-- def start_requests(self) -> Iterable[Request]:for url in self.sitemap_urls:yield Request(url, self._parse_sitemap)diff --git a/tests/test_spider_start.py b/tests/test_spider_start.pyindex 4e359fd33..aef2093ac 100644--- a/tests/test_spider_start.py+++ b/tests/test_spider_start.py@@ -1,15 +1,12 @@from __future__ import annotations-import reimport warningsfrom asyncio import sleepfrom typing import Anyimport pytest-from testfixtures import LogCapturefrom scrapy import Spider, signals-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.test import get_crawler@@ -77,81 +74,6 @@ class TestMain:warnings.simplefilter("error")await self._test_spider(TestSpider, [ITEM_A])- @coroutine_test- async def test_deprecated(self):- class TestSpider(Spider):- name = "test"-- def start_requests(self):- yield ITEM_A-- with pytest.warns(ScrapyDeprecationWarning):- await self._test_spider(TestSpider, [ITEM_A])-- @coroutine_test- async def test_deprecated_subclass(self):- class BaseSpider(Spider):- def start_requests(self):- yield ITEM_A-- class TestSpider(BaseSpider):- name = "test"-- # The warning must be about the base class and not the subclass.- with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"):- await self._test_spider(TestSpider, [ITEM_A])-- @coroutine_test- async def test_universal(self):- class TestSpider(Spider):- name = "test"-- async def start(self):- yield ITEM_A-- def start_requests(self):- yield ITEM_B-- with warnings.catch_warnings():- warnings.simplefilter("error")- await self._test_spider(TestSpider, [ITEM_A])-- @coroutine_test- async def test_universal_subclass(self):- class BaseSpider(Spider):- async def start(self):- yield ITEM_A-- def start_requests(self):- yield ITEM_B-- class TestSpider(BaseSpider):- name = "test"-- with warnings.catch_warnings():- warnings.simplefilter("error")- await self._test_spider(TestSpider, [ITEM_A])-- @coroutine_test- async def test_start_deprecated_super(self):- class TestSpider(Spider):- name = "test"-- async def start(self):- for item_or_request in super().start_requests():- yield item_or_request-- msg = "use Spider.start() instead"- with pytest.warns(ScrapyDeprecationWarning, match=re.escape(msg)) as ws:- await self._test_spider(TestSpider, [])-- for w in ws:- if isinstance(w.message, ScrapyDeprecationWarning) and msg in str(- w.message- ):- assert w.filename.endswith("test_spider_start.py")- break-async def _test_start(self, start_, expected_items=None):class TestSpider(Spider):name = "test"@@ -176,24 +98,3 @@ class TestMain:yield ITEM_Aawait self._test_start(start, [ITEM_A])-- # Exceptions-- @coroutine_test- async def test_deprecated_non_generator_exception(self):- class TestSpider(Spider):- name = "test"-- def start_requests(self):- raise RuntimeError-- with (- LogCapture() as log,- pytest.warns(- ScrapyDeprecationWarning,- match=r"defines the deprecated start_requests\(\) method",- ),- ):- await self._test_spider(TestSpider, [])-- assert "in start_requests\n raise RuntimeError" in str(log)diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.pyindex da570904d..70326f3f1 100644--- a/tests/test_spidermiddleware_base.py+++ b/tests/test_spidermiddleware_base.py@@ -7,7 +7,9 @@ import pytestfrom scrapy import Request, Spiderfrom scrapy.http import Responsefrom scrapy.spidermiddlewares.base import BaseSpiderMiddleware+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgenfrom scrapy.utils.test import get_crawler+from tests.utils.decorators import coroutine_testif TYPE_CHECKING:from scrapy.crawler import Crawler@@ -18,7 +20,8 @@ def crawler() -> Crawler:return get_crawler(Spider)-def test_trivial(crawler: Crawler) -> None:+@coroutine_test+async def test_trivial(crawler: Crawler) -> None:class TrivialSpiderMiddleware(BaseSpiderMiddleware):pass@@ -29,12 +32,13 @@ def test_trivial(crawler: Crawler) -> None:spider_output = [test_req, {"foo": "bar"}]for processed in [list(mw.process_spider_output(Response("data:,"), spider_output)),- list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]+ await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),]:assert processed == [test_req, {"foo": "bar"}]-def test_processed_request(crawler: Crawler) -> None:+@coroutine_test+async def test_processed_request(crawler: Crawler) -> None:class ProcessReqSpiderMiddleware(BaseSpiderMiddleware):def get_processed_request(self, request: Request, response: Response | None@@ -52,7 +56,7 @@ def test_processed_request(crawler: Crawler) -> None:spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]for processed in [list(mw.process_spider_output(Response("data:,"), spider_output)),- list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]+ await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),]:assert len(processed) == 3assert isinstance(processed[0], Request)@@ -62,7 +66,8 @@ def test_processed_request(crawler: Crawler) -> None:assert processed[2].url == "data:30,"-def test_processed_item(crawler: Crawler) -> None:+@coroutine_test+async def test_processed_item(crawler: Crawler) -> None:class ProcessItemSpiderMiddleware(BaseSpiderMiddleware):def get_processed_item(self, item: Any, response: Response | None) -> Any:if item["foo"] == 2:@@ -76,12 +81,13 @@ def test_processed_item(crawler: Crawler) -> None:spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]for processed in [list(mw.process_spider_output(Response("data:,"), spider_output)),- list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]+ await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),]:assert processed == [{"foo": 1}, test_req, {"foo": 30}]-def test_processed_both(crawler: Crawler) -> None:+@coroutine_test+async def test_processed_both(crawler: Crawler) -> None:class ProcessBothSpiderMiddleware(BaseSpiderMiddleware):def get_processed_request(self, request: Request, response: Response | None@@ -113,7 +119,7 @@ def test_processed_both(crawler: Crawler) -> None:]for processed in [list(mw.process_spider_output(Response("data:,"), spider_output)),- list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]+ await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),]:assert len(processed) == 4assert isinstance(processed[0], Request)diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.pyindex ddece52c4..c907c6d73 100644--- a/tests/test_spidermiddleware_process_start.py+++ b/tests/test_spidermiddleware_process_start.py@@ -4,7 +4,6 @@ from asyncio import sleepimport pytestfrom scrapy import Spider, signals-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.test import get_crawlerfrom tests.test_spider_start import SLEEP_SECONDS@@ -38,15 +37,6 @@ class TwistedSleepSpiderMiddleware:yield item_or_request-class UniversalSpiderMiddleware:- async def process_start(self, start):- async for item_or_request in start:- yield item_or_request-- def process_start_requests(self, start_requests, spider):- raise NotImplementedError--# Spiders and spider middlewares for TestMain._test_wrap@@ -61,23 +51,6 @@ class ModernWrapSpiderSubclass(ModernWrapSpider):name = "test"-class UniversalWrapSpider(Spider):- name = "test"-- async def start(self):- yield ITEM_B-- def start_requests(self):- yield ITEM_D---class DeprecatedWrapSpider(Spider):- name = "test"-- def start_requests(self):- yield ITEM_B--class ModernWrapSpiderMiddleware:async def process_start(self, start):yield ITEM_A@@ -86,26 +59,6 @@ class ModernWrapSpiderMiddleware:yield ITEM_C-class UniversalWrapSpiderMiddleware:- async def process_start(self, start):- yield ITEM_A- async for item_or_request in start:- yield item_or_request- yield ITEM_C-- def process_start_requests(self, start, spider):- yield ITEM_A- yield from start- yield ITEM_C---class DeprecatedWrapSpiderMiddleware:- def process_start_requests(self, start, spider):- yield ITEM_A- yield from start- yield ITEM_C--class TestMain:async def _test(self, spider_middlewares, spider_cls, expected_items):actual_items = []@@ -136,195 +89,6 @@ class TestMain:warnings.simplefilter("error")await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)- @coroutine_test- async def test_modern_mw_universal_spider(self):- with warnings.catch_warnings():- warnings.simplefilter("error")- await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider)-- @coroutine_test- async def test_modern_mw_deprecated_spider(self):- with pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"- ):- await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider)-- @coroutine_test- async def test_universal_mw_modern_spider(self):- with warnings.catch_warnings():- warnings.simplefilter("error")- await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider)-- @coroutine_test- async def test_universal_mw_universal_spider(self):- with warnings.catch_warnings():- warnings.simplefilter("error")- await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider)-- @coroutine_test- async def test_universal_mw_deprecated_spider(self):- with pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"- ):- await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider)-- @coroutine_test- async def test_deprecated_mw_modern_spider(self):- with (- pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"- ),- pytest.raises(- ValueError, match=r"only compatible with \(deprecated\) spiders"- ),- ):- await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider)-- @coroutine_test- async def test_deprecated_mw_modern_spider_subclass(self):- with (- pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"- ),- pytest.raises(- ValueError,- match=r"^\S+?\.ModernWrapSpider \(inherited by \S+?.ModernWrapSpiderSubclass\) .*? only compatible with \(deprecated\) spiders",- ),- ):- await self._test_wrap(- DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass- )-- @coroutine_test- async def test_deprecated_mw_universal_spider(self):- with pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"- ):- await self._test_wrap(- DeprecatedWrapSpiderMiddleware,- UniversalWrapSpider,- [ITEM_A, ITEM_D, ITEM_C],- )-- @coroutine_test- async def test_deprecated_mw_deprecated_spider(self):- with (- pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"- ),- pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"- ),- ):- await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider)-- @coroutine_test- async def test_modern_mw_universal_mw_modern_spider(self):- with warnings.catch_warnings():- warnings.simplefilter("error")- await self._test_douple_wrap(- ModernWrapSpiderMiddleware,- UniversalWrapSpiderMiddleware,- ModernWrapSpider,- )-- @coroutine_test- async def test_modern_mw_deprecated_mw_modern_spider(self):- with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):- await self._test_douple_wrap(- ModernWrapSpiderMiddleware,- DeprecatedWrapSpiderMiddleware,- ModernWrapSpider,- )-- @coroutine_test- async def test_universal_mw_deprecated_mw_modern_spider(self):- with (- pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"- ),- pytest.raises(- ValueError, match=r"only compatible with \(deprecated\) spiders"- ),- ):- await self._test_douple_wrap(- UniversalWrapSpiderMiddleware,- DeprecatedWrapSpiderMiddleware,- ModernWrapSpider,- )-- @coroutine_test- async def test_modern_mw_universal_mw_universal_spider(self):- with warnings.catch_warnings():- warnings.simplefilter("error")- await self._test_douple_wrap(- ModernWrapSpiderMiddleware,- UniversalWrapSpiderMiddleware,- UniversalWrapSpider,- )-- @coroutine_test- async def test_modern_mw_deprecated_mw_universal_spider(self):- with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):- await self._test_douple_wrap(- ModernWrapSpiderMiddleware,- DeprecatedWrapSpiderMiddleware,- UniversalWrapSpider,- )-- @coroutine_test- async def test_universal_mw_deprecated_mw_universal_spider(self):- with pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"- ):- await self._test_douple_wrap(- UniversalWrapSpiderMiddleware,- DeprecatedWrapSpiderMiddleware,- UniversalWrapSpider,- [ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C],- )-- @coroutine_test- async def test_modern_mw_universal_mw_deprecated_spider(self):- with pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"- ):- await self._test_douple_wrap(- ModernWrapSpiderMiddleware,- UniversalWrapSpiderMiddleware,- DeprecatedWrapSpider,- )-- @coroutine_test- async def test_modern_mw_deprecated_mw_deprecated_spider(self):- with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):- await self._test_douple_wrap(- ModernWrapSpiderMiddleware,- DeprecatedWrapSpiderMiddleware,- DeprecatedWrapSpider,- )-- @coroutine_test- async def test_universal_mw_deprecated_mw_deprecated_spider(self):- with (- pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"- ),- pytest.warns(- ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"- ),- ):- await self._test_douple_wrap(- UniversalWrapSpiderMiddleware,- DeprecatedWrapSpiderMiddleware,- DeprecatedWrapSpider,- )-- @coroutine_test- async def test_universal_mw_uses_process_start(self):- """Test that process_start_requests() isn't used when process_start() exists."""- await self._test([UniversalSpiderMiddleware], ModernWrapSpider, [ITEM_B])-async def _test_sleep(self, spider_middlewares):class TestSpider(Spider):name = "test"diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.pyindex 76976d962..26397d37c 100644--- a/tests/test_spidermiddleware_start.py+++ b/tests/test_spidermiddleware_start.py@@ -23,20 +23,3 @@ class TestMiddleware:async for request in mw.process_start(start())]assert result == [True, True, False, "foo"]-- @coroutine_test- async def test_sync(self):- crawler = get_crawler(Spider)- mw = build_from_crawler(StartSpiderMiddleware, crawler)-- def start():- yield Request("data:,1")- yield Request("data:,2", meta={"is_start_request": True})- yield Request("data:,2", meta={"is_start_request": False})- yield Request("data:,2", meta={"is_start_request": "foo"})-- result = [- request.meta["is_start_request"]- for request in mw.process_start_requests(start(), Spider("test"))- ]- assert result == [True, True, False, "foo"]