Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

fix(commands/parse): Restore request callback before invoking spider (#7803)

Laerte Pereira01447f996500
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 2ac65bf3f..93194ded7 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -346,6 +346,8 @@ class Command(BaseRunSpiderCommand):
self.first_response = response
cb = self._get_callback(spider=spider, opts=opts, response=response)
+ assert response.request
+ response.request.callback = cb
# parse items and requests
depth: int = response.meta["_depth"]
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 9b7131c7a..772cc82e2 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -126,6 +126,32 @@ class MySpider(scrapy.Spider):
else:
self.logger.debug('It Does Not Work :(')
+class RetryRequestSpider(BaseSpider):
+ name = 'retry_request'
+
+ def parse(self, response):
+ if response.meta.get('retried'):
+ yield {{'retried': True}}
+ return
+ response.meta['retried'] = True
+ yield response.request.replace(dont_filter=True)
+
+class CustomCallbackRetryRequestSpider(BaseSpider):
+ name = 'retry_request_custom_callback'
+
+ def parse(self, response):
+ yield response.request.replace(
+ callback=self.parse_retry,
+ dont_filter=True,
+ )
+
+ def parse_retry(self, response):
+ if response.meta.get('retried'):
+ yield {{'retried_with_custom_callback': True}}
+ return
+ response.meta['retried'] = True
+ yield response.request.replace(dont_filter=True)
+
class MyGoodCrawlSpider(CrawlSpider):
name = 'goodcrawl{self.spider_name}'
@@ -381,6 +407,36 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
)
assert "[{}, {'foo': 'bar'}]" in out
+ def test_retry_response_request(
+ self, proj_path: Path, mockserver: MockServer
+ ) -> None:
+ _, out, stderr = proc(
+ "parse",
+ "--spider",
+ "retry_request",
+ "-d",
+ "2",
+ mockserver.url("/html"),
+ cwd=proj_path,
+ )
+ assert "RecursionError" not in stderr
+ assert "{'retried': True}" in out
+
+ def test_retry_response_request_with_custom_callback(
+ self, proj_path: Path, mockserver: MockServer
+ ) -> None:
+ _, out, stderr = proc(
+ "parse",
+ "--spider",
+ "retry_request_custom_callback",
+ "-d",
+ "3",
+ mockserver.url("/html"),
+ cwd=proj_path,
+ )
+ assert "RecursionError" not in stderr
+ assert "{'retried_with_custom_callback': True}" in out
+
def test_wrong_callback_passed(
self, proj_path: Path, mockserver: MockServer
) -> None: