Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'origin/master' into asyncio-parse-asyncgen-proper-rebased
Andrey Rakhmatullinffca87ae88cd
diff --cc scrapy/core/scraper.pyindex 0630ce625,96aa53686..71086667e--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@@ -1,11 -1,12 +1,11 @@@"""This module implements the Scraper component which parses responses andextracts information from them"""-import loggingfrom collections import deque- from typing import AsyncGenerator, AsyncIterable, Generator, Iterable, Union-from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union++from typing import Any, AsyncGenerator, AsyncIterable, Deque, Generator, Iterable, Optional, Set, Tuple, Unionfrom itemadapter import is_item- from twisted.internet import defer+ from twisted.internet.defer import Deferred, inlineCallbacksfrom twisted.python.failure import Failurefrom scrapy import signals, Spider@@@ -183,21 -185,16 +192,22 @@@ class Scraperspider=spider)- def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider):+ def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred:if not result:return defer_succeed(None)- it = iter_errback(result, self.handle_spider_error, request, response, spider)- dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,- request, response, spider)+ it: Union[Generator, AsyncGenerator]+ if isinstance(result, AsyncIterable):+ it = aiter_errback(result, self.handle_spider_error, request, response, spider)+ dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,+ request, response, spider)+ else:+ it = iter_errback(result, self.handle_spider_error, request, response, spider)+ dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,+ request, response, spider)return dfd- def _process_spidermw_output(self, output, request, response, spider):+ def _process_spidermw_output(self, output: Any, request: Request, response: Response,+ spider: Spider) -> Optional[Deferred]:"""Process each Request/Item (given in the output parameter) returnedfrom the given spider"""