Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Drop start_requests() support. (#7490)

Andrey Rakhmatullinf3868e11fb0e
diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index 269b27336..8e5001c1d 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -35,10 +35,6 @@ class QPSSpider(Spider):
self.download_delay = float(self.download_delay)
async def start(self):
- for item_or_request in self.start_requests():
- yield item_or_request
-
- def start_requests(self):
url = self.benchurl
if self.latency is not None:
url += f"?latency={self.latency}"
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 67342099d..790317357 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -33,7 +33,6 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_n
if TYPE_CHECKING:
from collections.abc import Generator
- from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings
@@ -60,76 +59,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
settings.get_component_priority_dict_with_base("SPIDER_MIDDLEWARES")
)
- def __init__(self, *middlewares: Any, crawler: Crawler | None = None) -> None:
- self._check_deprecated_process_start_requests_use(middlewares)
- super().__init__(*middlewares, crawler=crawler)
-
- def _check_deprecated_process_start_requests_use(
- self, middlewares: tuple[Any, ...]
- ) -> None:
- deprecated_middlewares = [
- middleware
- for middleware in middlewares
- if hasattr(middleware, "process_start_requests")
- and not hasattr(middleware, "process_start")
- ]
- modern_middlewares = [
- middleware
- for middleware in middlewares
- if not hasattr(middleware, "process_start_requests")
- and hasattr(middleware, "process_start")
- ]
- if deprecated_middlewares and modern_middlewares:
- raise ValueError(
- "You are trying to combine spider middlewares that only "
- "define the deprecated process_start_requests() method () "
- "with spider middlewares that only define the "
- "process_start() method (). This is not possible. You must "
- "either disable or make universal 1 of those 2 sets of "
- "spider middlewares. Making a spider middleware universal "
- "means having it define both methods. See the release notes "
- "of Scrapy 2.13 for details: "
- "https://docs.scrapy.org/en/2.13/news.html"
- )
-
- self._use_start_requests = bool(deprecated_middlewares)
- if self._use_start_requests:
- deprecated_middleware_list = ", ".join(
- global_object_name(middleware.__class__)
- for middleware in deprecated_middlewares
- )
- warn(
- f"The following enabled spider middlewares, directly or "
- f"through their parent classes, define the deprecated "
- f"process_start_requests() method: "
- f"{deprecated_middleware_list}. process_start_requests() has "
- f"been deprecated in favor of a new method, process_start(), "
- f"to support asynchronous code execution. "
- f"process_start_requests() will stop being called in a future "
- f"version of Scrapy. If you use Scrapy 2.13 or higher "
- f"only, replace process_start_requests() with "
- f"process_start(); note that process_start() is a coroutine "
- f"(async def). If you need to maintain compatibility with "
- f"lower Scrapy versions, when defining "
- f"process_start_requests() in a spider middleware class, "
- f"define process_start() as well. See the release notes of "
- f"Scrapy 2.13 for details: "
- f"https://docs.scrapy.org/en/2.13/news.html",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
-
def _add_middleware(self, mw: Any) -> None:
if hasattr(mw, "process_spider_input"):
self.methods["process_spider_input"].append(mw.process_spider_input)
self._check_mw_method_spider_arg(mw.process_spider_input)
- if self._use_start_requests:
- if hasattr(mw, "process_start_requests"):
- self.methods["process_start_requests"].appendleft(
- mw.process_start_requests
- )
- elif hasattr(mw, "process_start"):
+ if hasattr(mw, "process_start"):
self.methods["process_start"].appendleft(mw.process_start)
process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
@@ -451,88 +386,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
)
warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
self._set_compat_spider(spider)
- self._check_deprecated_start_requests_use()
- if self._use_start_requests:
- sync_start = iter(self._spider.start_requests())
- sync_start = await self._process_chain(
- "process_start_requests", sync_start, always_add_spider=True
- )
- start: AsyncIterator[Any] = as_async_generator(sync_start)
- else:
- start = self._spider.start()
- start = await self._process_chain("process_start", start)
- return start
-
- def _check_deprecated_start_requests_use(self) -> None:
- start_requests_cls = None
- start_cls = None
- spidercls = self._spider.__class__
- mro = spidercls.__mro__
-
- for cls in mro:
- cls_dict = cls.__dict__
- if start_requests_cls is None and "start_requests" in cls_dict:
- start_requests_cls = cls
- if start_cls is None and "start" in cls_dict:
- start_cls = cls
- if start_requests_cls is not None and start_cls is not None:
- break
-
- # Spider defines both, start_requests and start.
- assert start_requests_cls is not None
- assert start_cls is not None
-
- if (
- start_requests_cls is not Spider
- and start_cls is not start_requests_cls
- and mro.index(start_requests_cls) < mro.index(start_cls)
- ):
- src = global_object_name(start_requests_cls)
- if start_requests_cls is not spidercls:
- src += f" (inherited by {global_object_name(spidercls)})"
- warn(
- f"{src} defines the deprecated start_requests() method. "
- f"start_requests() has been deprecated in favor of a new "
- f"method, start(), to support asynchronous code "
- f"execution. start_requests() will stop being called in a "
- f"future version of Scrapy. If you use Scrapy 2.13 or "
- f"higher only, replace start_requests() with start(); "
- f"note that start() is a coroutine (async def). If you "
- f"need to maintain compatibility with lower Scrapy versions, "
- f"when overriding start_requests() in a spider class, "
- f"override start() as well; you can use super() to "
- f"reuse the inherited start() implementation without "
- f"copy-pasting. See the release notes of Scrapy 2.13 for "
- f"details: https://docs.scrapy.org/en/2.13/news.html",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
-
- if (
- self._use_start_requests
- and start_cls is not Spider
- and start_requests_cls is not start_cls
- and mro.index(start_cls) < mro.index(start_requests_cls)
- ):
- src = global_object_name(start_cls)
- if start_cls is not spidercls:
- src += f" (inherited by {global_object_name(spidercls)})"
- raise ValueError(
- f"{src} does not define the deprecated start_requests() "
- f"method. However, one or more of your enabled spider "
- f"middlewares (reported in an earlier deprecation warning) "
- f"define the process_start_requests() method, and not the "
- f"process_start() method, making them only compatible with "
- f"(deprecated) spiders that define the start_requests() "
- f"method. To solve this issue, disable the offending spider "
- f"middlewares, upgrade them as described in that earlier "
- f"deprecation warning, or make your spider compatible with "
- f"deprecated spider middlewares (and earlier Scrapy versions) "
- f"by defining a sync start_requests() method that works "
- f"similarly to its existing start() method. See the "
- f"release notes of Scrapy 2.13 for details: "
- f"https://docs.scrapy.org/en/2.13/news.html"
- )
+ start = self._spider.start()
+ return await self._process_chain("process_start", start)
# This method is only needed until _async compatibility methods are removed.
@staticmethod
diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py
index 889fc6df1..e09f2d10e 100644
--- a/scrapy/spidermiddlewares/base.py
+++ b/scrapy/spidermiddlewares/base.py
@@ -41,13 +41,6 @@ class BaseSpiderMiddleware:
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
- def process_start_requests(
- self, start: Iterable[Any], spider: Spider
- ) -> Iterable[Any]:
- for o in start:
- if (o := self._get_processed(o, None)) is not None:
- yield o
-
async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
async for o in start:
if (o := self._get_processed(o, None)) is not None:
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 018e510c5..299a5d43f 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -7,17 +7,15 @@ See documentation in docs/topics/spiders.rst
from __future__ import annotations
import logging
-import warnings
from typing import TYPE_CHECKING, Any, cast
from scrapy import signals
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
if TYPE_CHECKING:
- from collections.abc import AsyncIterator, Iterable
+ from collections.abc import AsyncIterator
from twisted.internet.defer import Deferred
@@ -127,24 +125,6 @@ class Spider(object_ref):
.. seealso:: :ref:`start-requests`
"""
- with warnings.catch_warnings():
- warnings.filterwarnings(
- "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$"
- )
- for item_or_request in self.start_requests():
- yield item_or_request
-
- def start_requests(self) -> Iterable[Any]:
- warnings.warn(
- (
- "The Spider.start_requests() method is deprecated, use "
- "Spider.start() instead. If you are calling "
- "super().start_requests() from a Spider.start() override, "
- "iterate super().start() instead."
- ),
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
if not self.start_urls and hasattr(self, "start_url"):
raise AttributeError(
"Crawling could not start: 'start_urls' not found "
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index d87ffccaf..2a80b8d24 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -54,10 +54,6 @@ class SitemapSpider(Spider):
self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]
async def start(self) -> AsyncIterator[Any]:
- for item_or_request in self.start_requests():
- yield item_or_request
-
- def start_requests(self) -> Iterable[Request]:
for url in self.sitemap_urls:
yield Request(url, self._parse_sitemap)
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index 4e359fd33..aef2093ac 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -1,15 +1,12 @@
from __future__ import annotations
-import re
import warnings
from asyncio import sleep
from typing import Any
import pytest
-from testfixtures import LogCapture
from scrapy import Spider, signals
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
@@ -77,81 +74,6 @@ class TestMain:
warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A])
- @coroutine_test
- async def test_deprecated(self):
- class TestSpider(Spider):
- name = "test"
-
- def start_requests(self):
- yield ITEM_A
-
- with pytest.warns(ScrapyDeprecationWarning):
- await self._test_spider(TestSpider, [ITEM_A])
-
- @coroutine_test
- async def test_deprecated_subclass(self):
- class BaseSpider(Spider):
- def start_requests(self):
- yield ITEM_A
-
- class TestSpider(BaseSpider):
- name = "test"
-
- # The warning must be about the base class and not the subclass.
- with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"):
- await self._test_spider(TestSpider, [ITEM_A])
-
- @coroutine_test
- async def test_universal(self):
- class TestSpider(Spider):
- name = "test"
-
- async def start(self):
- yield ITEM_A
-
- def start_requests(self):
- yield ITEM_B
-
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- await self._test_spider(TestSpider, [ITEM_A])
-
- @coroutine_test
- async def test_universal_subclass(self):
- class BaseSpider(Spider):
- async def start(self):
- yield ITEM_A
-
- def start_requests(self):
- yield ITEM_B
-
- class TestSpider(BaseSpider):
- name = "test"
-
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- await self._test_spider(TestSpider, [ITEM_A])
-
- @coroutine_test
- async def test_start_deprecated_super(self):
- class TestSpider(Spider):
- name = "test"
-
- async def start(self):
- for item_or_request in super().start_requests():
- yield item_or_request
-
- msg = "use Spider.start() instead"
- with pytest.warns(ScrapyDeprecationWarning, match=re.escape(msg)) as ws:
- await self._test_spider(TestSpider, [])
-
- for w in ws:
- if isinstance(w.message, ScrapyDeprecationWarning) and msg in str(
- w.message
- ):
- assert w.filename.endswith("test_spider_start.py")
- break
-
async def _test_start(self, start_, expected_items=None):
class TestSpider(Spider):
name = "test"
@@ -176,24 +98,3 @@ class TestMain:
yield ITEM_A
await self._test_start(start, [ITEM_A])
-
- # Exceptions
-
- @coroutine_test
- async def test_deprecated_non_generator_exception(self):
- class TestSpider(Spider):
- name = "test"
-
- def start_requests(self):
- raise RuntimeError
-
- with (
- LogCapture() as log,
- pytest.warns(
- ScrapyDeprecationWarning,
- match=r"defines the deprecated start_requests\(\) method",
- ),
- ):
- await self._test_spider(TestSpider, [])
-
- assert "in start_requests\n raise RuntimeError" in str(log)
diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py
index da570904d..70326f3f1 100644
--- a/tests/test_spidermiddleware_base.py
+++ b/tests/test_spidermiddleware_base.py
@@ -7,7 +7,9 @@ import pytest
from scrapy import Request, Spider
from scrapy.http import Response
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.crawler import Crawler
@@ -18,7 +20,8 @@ def crawler() -> Crawler:
return get_crawler(Spider)
-def test_trivial(crawler: Crawler) -> None:
+@coroutine_test
+async def test_trivial(crawler: Crawler) -> None:
class TrivialSpiderMiddleware(BaseSpiderMiddleware):
pass
@@ -29,12 +32,13 @@ def test_trivial(crawler: Crawler) -> None:
spider_output = [test_req, {"foo": "bar"}]
for processed in [
list(mw.process_spider_output(Response("data:,"), spider_output)),
- list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
+ await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
]:
assert processed == [test_req, {"foo": "bar"}]
-def test_processed_request(crawler: Crawler) -> None:
+@coroutine_test
+async def test_processed_request(crawler: Crawler) -> None:
class ProcessReqSpiderMiddleware(BaseSpiderMiddleware):
def get_processed_request(
self, request: Request, response: Response | None
@@ -52,7 +56,7 @@ def test_processed_request(crawler: Crawler) -> None:
spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]
for processed in [
list(mw.process_spider_output(Response("data:,"), spider_output)),
- list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
+ await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
]:
assert len(processed) == 3
assert isinstance(processed[0], Request)
@@ -62,7 +66,8 @@ def test_processed_request(crawler: Crawler) -> None:
assert processed[2].url == "data:30,"
-def test_processed_item(crawler: Crawler) -> None:
+@coroutine_test
+async def test_processed_item(crawler: Crawler) -> None:
class ProcessItemSpiderMiddleware(BaseSpiderMiddleware):
def get_processed_item(self, item: Any, response: Response | None) -> Any:
if item["foo"] == 2:
@@ -76,12 +81,13 @@ def test_processed_item(crawler: Crawler) -> None:
spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]
for processed in [
list(mw.process_spider_output(Response("data:,"), spider_output)),
- list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
+ await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
]:
assert processed == [{"foo": 1}, test_req, {"foo": 30}]
-def test_processed_both(crawler: Crawler) -> None:
+@coroutine_test
+async def test_processed_both(crawler: Crawler) -> None:
class ProcessBothSpiderMiddleware(BaseSpiderMiddleware):
def get_processed_request(
self, request: Request, response: Response | None
@@ -113,7 +119,7 @@ def test_processed_both(crawler: Crawler) -> None:
]
for processed in [
list(mw.process_spider_output(Response("data:,"), spider_output)),
- list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
+ await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
]:
assert len(processed) == 4
assert isinstance(processed[0], Request)
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index ddece52c4..c907c6d73 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -4,7 +4,6 @@ from asyncio import sleep
import pytest
from scrapy import Spider, signals
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from tests.test_spider_start import SLEEP_SECONDS
@@ -38,15 +37,6 @@ class TwistedSleepSpiderMiddleware:
yield item_or_request
-class UniversalSpiderMiddleware:
- async def process_start(self, start):
- async for item_or_request in start:
- yield item_or_request
-
- def process_start_requests(self, start_requests, spider):
- raise NotImplementedError
-
-
# Spiders and spider middlewares for TestMain._test_wrap
@@ -61,23 +51,6 @@ class ModernWrapSpiderSubclass(ModernWrapSpider):
name = "test"
-class UniversalWrapSpider(Spider):
- name = "test"
-
- async def start(self):
- yield ITEM_B
-
- def start_requests(self):
- yield ITEM_D
-
-
-class DeprecatedWrapSpider(Spider):
- name = "test"
-
- def start_requests(self):
- yield ITEM_B
-
-
class ModernWrapSpiderMiddleware:
async def process_start(self, start):
yield ITEM_A
@@ -86,26 +59,6 @@ class ModernWrapSpiderMiddleware:
yield ITEM_C
-class UniversalWrapSpiderMiddleware:
- async def process_start(self, start):
- yield ITEM_A
- async for item_or_request in start:
- yield item_or_request
- yield ITEM_C
-
- def process_start_requests(self, start, spider):
- yield ITEM_A
- yield from start
- yield ITEM_C
-
-
-class DeprecatedWrapSpiderMiddleware:
- def process_start_requests(self, start, spider):
- yield ITEM_A
- yield from start
- yield ITEM_C
-
-
class TestMain:
async def _test(self, spider_middlewares, spider_cls, expected_items):
actual_items = []
@@ -136,195 +89,6 @@ class TestMain:
warnings.simplefilter("error")
await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
- @coroutine_test
- async def test_modern_mw_universal_spider(self):
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider)
-
- @coroutine_test
- async def test_modern_mw_deprecated_spider(self):
- with pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
- ):
- await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider)
-
- @coroutine_test
- async def test_universal_mw_modern_spider(self):
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider)
-
- @coroutine_test
- async def test_universal_mw_universal_spider(self):
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider)
-
- @coroutine_test
- async def test_universal_mw_deprecated_spider(self):
- with pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
- ):
- await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider)
-
- @coroutine_test
- async def test_deprecated_mw_modern_spider(self):
- with (
- pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
- ),
- pytest.raises(
- ValueError, match=r"only compatible with \(deprecated\) spiders"
- ),
- ):
- await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider)
-
- @coroutine_test
- async def test_deprecated_mw_modern_spider_subclass(self):
- with (
- pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
- ),
- pytest.raises(
- ValueError,
- match=r"^\S+?\.ModernWrapSpider \(inherited by \S+?.ModernWrapSpiderSubclass\) .*? only compatible with \(deprecated\) spiders",
- ),
- ):
- await self._test_wrap(
- DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass
- )
-
- @coroutine_test
- async def test_deprecated_mw_universal_spider(self):
- with pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
- ):
- await self._test_wrap(
- DeprecatedWrapSpiderMiddleware,
- UniversalWrapSpider,
- [ITEM_A, ITEM_D, ITEM_C],
- )
-
- @coroutine_test
- async def test_deprecated_mw_deprecated_spider(self):
- with (
- pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
- ),
- pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
- ),
- ):
- await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider)
-
- @coroutine_test
- async def test_modern_mw_universal_mw_modern_spider(self):
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- await self._test_douple_wrap(
- ModernWrapSpiderMiddleware,
- UniversalWrapSpiderMiddleware,
- ModernWrapSpider,
- )
-
- @coroutine_test
- async def test_modern_mw_deprecated_mw_modern_spider(self):
- with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
- await self._test_douple_wrap(
- ModernWrapSpiderMiddleware,
- DeprecatedWrapSpiderMiddleware,
- ModernWrapSpider,
- )
-
- @coroutine_test
- async def test_universal_mw_deprecated_mw_modern_spider(self):
- with (
- pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
- ),
- pytest.raises(
- ValueError, match=r"only compatible with \(deprecated\) spiders"
- ),
- ):
- await self._test_douple_wrap(
- UniversalWrapSpiderMiddleware,
- DeprecatedWrapSpiderMiddleware,
- ModernWrapSpider,
- )
-
- @coroutine_test
- async def test_modern_mw_universal_mw_universal_spider(self):
- with warnings.catch_warnings():
- warnings.simplefilter("error")
- await self._test_douple_wrap(
- ModernWrapSpiderMiddleware,
- UniversalWrapSpiderMiddleware,
- UniversalWrapSpider,
- )
-
- @coroutine_test
- async def test_modern_mw_deprecated_mw_universal_spider(self):
- with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
- await self._test_douple_wrap(
- ModernWrapSpiderMiddleware,
- DeprecatedWrapSpiderMiddleware,
- UniversalWrapSpider,
- )
-
- @coroutine_test
- async def test_universal_mw_deprecated_mw_universal_spider(self):
- with pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
- ):
- await self._test_douple_wrap(
- UniversalWrapSpiderMiddleware,
- DeprecatedWrapSpiderMiddleware,
- UniversalWrapSpider,
- [ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C],
- )
-
- @coroutine_test
- async def test_modern_mw_universal_mw_deprecated_spider(self):
- with pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
- ):
- await self._test_douple_wrap(
- ModernWrapSpiderMiddleware,
- UniversalWrapSpiderMiddleware,
- DeprecatedWrapSpider,
- )
-
- @coroutine_test
- async def test_modern_mw_deprecated_mw_deprecated_spider(self):
- with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
- await self._test_douple_wrap(
- ModernWrapSpiderMiddleware,
- DeprecatedWrapSpiderMiddleware,
- DeprecatedWrapSpider,
- )
-
- @coroutine_test
- async def test_universal_mw_deprecated_mw_deprecated_spider(self):
- with (
- pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
- ),
- pytest.warns(
- ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
- ),
- ):
- await self._test_douple_wrap(
- UniversalWrapSpiderMiddleware,
- DeprecatedWrapSpiderMiddleware,
- DeprecatedWrapSpider,
- )
-
- @coroutine_test
- async def test_universal_mw_uses_process_start(self):
- """Test that process_start_requests() isn't used when process_start() exists."""
- await self._test([UniversalSpiderMiddleware], ModernWrapSpider, [ITEM_B])
-
async def _test_sleep(self, spider_middlewares):
class TestSpider(Spider):
name = "test"
diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py
index 76976d962..26397d37c 100644
--- a/tests/test_spidermiddleware_start.py
+++ b/tests/test_spidermiddleware_start.py
@@ -23,20 +23,3 @@ class TestMiddleware:
async for request in mw.process_start(start())
]
assert result == [True, True, False, "foo"]
-
- @coroutine_test
- async def test_sync(self):
- crawler = get_crawler(Spider)
- mw = build_from_crawler(StartSpiderMiddleware, crawler)
-
- def start():
- yield Request("data:,1")
- yield Request("data:,2", meta={"is_start_request": True})
- yield Request("data:,2", meta={"is_start_request": False})
- yield Request("data:,2", meta={"is_start_request": "foo"})
-
- result = [
- request.meta["is_start_request"]
- for request in mw.process_start_requests(start(), Spider("test"))
- ]
- assert result == [True, True, False, "foo"]