Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'origin/master' into asyncio-parse-asyncgen-proper-rebased

Andrey Rakhmatullinffca87ae88cd
diff --cc scrapy/core/scraper.py
index 0630ce625,96aa53686..71086667e
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@@ -1,11 -1,12 +1,11 @@@
"""This module implements the Scraper component which parses responses and
extracts information from them"""
-
import logging
from collections import deque
- from typing import AsyncGenerator, AsyncIterable, Generator, Iterable, Union
-from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union
++from typing import Any, AsyncGenerator, AsyncIterable, Deque, Generator, Iterable, Optional, Set, Tuple, Union
from itemadapter import is_item
- from twisted.internet import defer
+ from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.python.failure import Failure
from scrapy import signals, Spider
@@@ -183,21 -185,16 +192,22 @@@ class Scraper
spider=spider
)
- def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider):
+ def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred:
if not result:
return defer_succeed(None)
- it = iter_errback(result, self.handle_spider_error, request, response, spider)
- dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
- request, response, spider)
+ it: Union[Generator, AsyncGenerator]
+ if isinstance(result, AsyncIterable):
+ it = aiter_errback(result, self.handle_spider_error, request, response, spider)
+ dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,
+ request, response, spider)
+ else:
+ it = iter_errback(result, self.handle_spider_error, request, response, spider)
+ dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
+ request, response, spider)
return dfd
- def _process_spidermw_output(self, output, request, response, spider):
+ def _process_spidermw_output(self, output: Any, request: Request, response: Response,
+ spider: Spider) -> Optional[Deferred]:
"""Process each Request/Item (given in the output parameter) returned
from the given spider
"""