Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Init redirect middlewares in spider_opened instead of engine_started. (#8174)

Andrey Rakhmatullin5e30da4601fa
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 45af5c67a..29815d593 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -43,7 +43,7 @@ class BaseRedirectMiddleware:
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls(crawler.settings)
o.crawler = crawler
- crawler.signals.connect(o._engine_started, signal=signals.engine_started)
+ crawler.signals.connect(o._spider_opened, signal=signals.spider_opened)
return o
def handle_referer(self, request: Request, response: Response) -> None:
@@ -63,7 +63,7 @@ class BaseRedirectMiddleware:
return
self._referer_spider_middleware.get_processed_request(request, response)
- def _engine_started(self) -> None:
+ def _spider_opened(self) -> None:
self._referer_spider_middleware = self.crawler.get_spider_middleware(
RefererMiddleware
)
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 4cebcb281..180c1a669 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -418,7 +418,7 @@ def test_no_warning_when_referer_middleware_present(caplog):
mw = build_from_crawler(RedirectMiddleware, crawler)
caplog.clear()
with caplog.at_level(logging.WARNING):
- mw._engine_started()
+ mw._spider_opened()
assert not [
record
for record in caplog.records
@@ -433,7 +433,7 @@ def test_warning_redirect_middleware(caplog):
)
mw = build_from_crawler(RedirectMiddleware, crawler)
with caplog.at_level(logging.WARNING):
- mw._engine_started()
+ mw._spider_opened()
assert (
"scrapy.downloadermiddlewares.redirect.RedirectMiddleware found no "
"scrapy.spidermiddlewares.referer.RefererMiddleware"
@@ -458,7 +458,7 @@ def test_warning_subclass(caplog):
)
mw = build_from_crawler(MyRedirectMiddleware, crawler)
with caplog.at_level(logging.WARNING):
- mw._engine_started()
+ mw._spider_opened()
assert (
"test_warning_subclass.<locals>.MyRedirectMiddleware found no "
"scrapy.spidermiddlewares.referer.RefererMiddleware"
diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py
index 3887ddd26..bee79ba14 100644
--- a/tests/test_downloadermiddleware_redirect_metarefresh.py
+++ b/tests/test_downloadermiddleware_redirect_metarefresh.py
@@ -151,7 +151,7 @@ def test_warning_meta_refresh_middleware(caplog):
)
mw = build_from_crawler(MetaRefreshMiddleware, crawler)
with caplog.at_level(logging.WARNING):
- mw._engine_started()
+ mw._spider_opened()
assert (
"scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware found no "
"scrapy.spidermiddlewares.referer.RefererMiddleware"