Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'origin/master' into deferred-typing

Andrey Rakhmatullin5b547f08085e
diff --cc scrapy/core/scraper.py
index 4a3eff888,4a3ddea10..c760a4155
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@@ -114,15 -110,15 +112,15 @@@ class Scraper
{'request': request},
exc_info=failure_to_exc_info(f),
extra={'spider': spider}))
- self._scrape_next(spider, slot)
+ self._scrape_next(spider)
return dfd
- def _scrape_next(self, spider, slot):
- while slot.queue:
- response, request, deferred = slot.next_response_request_deferred()
+ def _scrape_next(self, spider):
+ while self.slot.queue:
+ response, request, deferred = self.slot.next_response_request_deferred()
self._scrape(response, request, spider).chainDeferred(deferred)
- def _scrape(self, result, request, spider):
+ def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider):
"""
Handle the downloaded response or failure through the spider callback/errback
"""