Scrapy, a fast high-level web crawling & scraping framework for Python.
fix(commands/parse): Restore request callback before invoking spider (#7803)
Laerte Pereira01447f996500
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex 2ac65bf3f..93194ded7 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -346,6 +346,8 @@ class Command(BaseRunSpiderCommand):self.first_response = responsecb = self._get_callback(spider=spider, opts=opts, response=response)+ assert response.request+ response.request.callback = cb# parse items and requestsdepth: int = response.meta["_depth"]diff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex 9b7131c7a..772cc82e2 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -126,6 +126,32 @@ class MySpider(scrapy.Spider):else:self.logger.debug('It Does Not Work :(')+class RetryRequestSpider(BaseSpider):+ name = 'retry_request'++ def parse(self, response):+ if response.meta.get('retried'):+ yield {{'retried': True}}+ return+ response.meta['retried'] = True+ yield response.request.replace(dont_filter=True)++class CustomCallbackRetryRequestSpider(BaseSpider):+ name = 'retry_request_custom_callback'++ def parse(self, response):+ yield response.request.replace(+ callback=self.parse_retry,+ dont_filter=True,+ )++ def parse_retry(self, response):+ if response.meta.get('retried'):+ yield {{'retried_with_custom_callback': True}}+ return+ response.meta['retried'] = True+ yield response.request.replace(dont_filter=True)+class MyGoodCrawlSpider(CrawlSpider):name = 'goodcrawl{self.spider_name}'@@ -381,6 +407,36 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}})assert "[{}, {'foo': 'bar'}]" in out+ def test_retry_response_request(+ self, proj_path: Path, mockserver: MockServer+ ) -> None:+ _, out, stderr = proc(+ "parse",+ "--spider",+ "retry_request",+ "-d",+ "2",+ mockserver.url("/html"),+ cwd=proj_path,+ )+ assert "RecursionError" not in stderr+ assert "{'retried': True}" in out++ def test_retry_response_request_with_custom_callback(+ self, proj_path: Path, mockserver: MockServer+ ) -> None:+ _, out, stderr = proc(+ "parse",+ "--spider",+ "retry_request_custom_callback",+ "-d",+ "3",+ mockserver.url("/html"),+ cwd=proj_path,+ )+ assert "RecursionError" not in stderr+ assert "{'retried_with_custom_callback': True}" in out+def test_wrong_callback_passed(self, proj_path: Path, mockserver: MockServer) -> None: