Scrapy, a fast high-level web crawling & scraping framework for Python.
Init redirect middlewares in spider_opened instead of engine_started. (#8174)
Andrey Rakhmatullin5e30da4601fa
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 45af5c67a..29815d593 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -43,7 +43,7 @@ class BaseRedirectMiddleware:def from_crawler(cls, crawler: Crawler) -> Self:o = cls(crawler.settings)o.crawler = crawler- crawler.signals.connect(o._engine_started, signal=signals.engine_started)+ crawler.signals.connect(o._spider_opened, signal=signals.spider_opened)return odef handle_referer(self, request: Request, response: Response) -> None:@@ -63,7 +63,7 @@ class BaseRedirectMiddleware:returnself._referer_spider_middleware.get_processed_request(request, response)- def _engine_started(self) -> None:+ def _spider_opened(self) -> None:self._referer_spider_middleware = self.crawler.get_spider_middleware(RefererMiddleware)diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 4cebcb281..180c1a669 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -418,7 +418,7 @@ def test_no_warning_when_referer_middleware_present(caplog):mw = build_from_crawler(RedirectMiddleware, crawler)caplog.clear()with caplog.at_level(logging.WARNING):- mw._engine_started()+ mw._spider_opened()assert not [recordfor record in caplog.records@@ -433,7 +433,7 @@ def test_warning_redirect_middleware(caplog):)mw = build_from_crawler(RedirectMiddleware, crawler)with caplog.at_level(logging.WARNING):- mw._engine_started()+ mw._spider_opened()assert ("scrapy.downloadermiddlewares.redirect.RedirectMiddleware found no ""scrapy.spidermiddlewares.referer.RefererMiddleware"@@ -458,7 +458,7 @@ def test_warning_subclass(caplog):)mw = build_from_crawler(MyRedirectMiddleware, crawler)with caplog.at_level(logging.WARNING):- mw._engine_started()+ mw._spider_opened()assert ("test_warning_subclass.<locals>.MyRedirectMiddleware found no ""scrapy.spidermiddlewares.referer.RefererMiddleware"diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.pyindex 3887ddd26..bee79ba14 100644--- a/tests/test_downloadermiddleware_redirect_metarefresh.py+++ b/tests/test_downloadermiddleware_redirect_metarefresh.py@@ -151,7 +151,7 @@ def test_warning_meta_refresh_middleware(caplog):)mw = build_from_crawler(MetaRefreshMiddleware, crawler)with caplog.at_level(logging.WARNING):- mw._engine_started()+ mw._spider_opened()assert ("scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware found no ""scrapy.spidermiddlewares.referer.RefererMiddleware"