Scrapy, a fast high-level web crawling & scraping framework for Python.
refactor(yield): use `yield from` syntax
Rotzbua0d445a3224ec
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex dcf1a6dbc..031a0be36 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -103,8 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):) -> Union[Generator, AsyncGenerator]:def process_sync(iterable: Iterable) -> Generator:try:- for r in iterable:- yield r+ yield from iterableexcept Exception as ex:exception_result = self._process_spider_exception(response, spider, Failure(ex), exception_processor_indexdiff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex 31e845716..ebb4f5984 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -131,8 +131,7 @@ class CrawlSpider(Spider):def _handle_failure(self, failure, errback):if errback:results = errback(failure) or ()- for request_or_item in iterate_spider_output(results):- yield request_or_item+ yield from iterate_spider_output(results)def _compile_rules(self):self._rules = []diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.pyindex 6afadc577..47827e442 100644--- a/scrapy/spiders/feed.py+++ b/scrapy/spiders/feed.py@@ -58,8 +58,7 @@ class XMLFeedSpider(Spider):for selector in nodes:ret = iterate_spider_output(self.parse_node(response, selector))- for result_item in self.process_results(response, ret):- yield result_item+ yield from self.process_results(response, ret)def _parse(self, response, **kwargs):if not hasattr(self, "parse_node"):@@ -133,8 +132,7 @@ class CSVFeedSpider(Spider):response, self.delimiter, self.headers, quotechar=self.quotechar):ret = iterate_spider_output(self.parse_row(response, row))- for result_item in self.process_results(response, ret):- yield result_item+ yield from self.process_results(response, ret)def _parse(self, response, **kwargs):if not hasattr(self, "parse_row"):diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex aaf75a519..974665fe0 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -33,8 +33,7 @@ class SitemapSpider(Spider):attributes, for example, you can filter locs with lastmod greaterthan a given date (see docs)."""- for entry in entries:- yield entry+ yield from entriesdef _parse_sitemap(self, response):if response.url.endswith("/robots.txt"):diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 0b5dc324f..68ca96b69 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -57,8 +57,7 @@ def iflatten(x: Iterable) -> Iterable:Similar to ``.flatten()``, but returns iterator instead"""for el in x:if is_listlike(el):- for el_ in iflatten(el):- yield el_+ yield from iflatten(el)else:yield eldiff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 24fcbd85e..cea1bc727 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -44,8 +44,7 @@ def _serialize_headers(for header in headers:if header in request.headers:yield header- for value in request.headers.getlist(header):- yield value+ yield from request.headers.getlist(header)def request_fingerprint(diff --git a/tests/spiders.py b/tests/spiders.pyindex f29dea2a1..3df153a12 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -301,8 +301,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):def parse(self, response):self.seedsseen.append(response.meta.get("seed"))- for req in super().parse(response):- yield req+ yield from super().parse(response)class SingleRequestSpider(MetaSpider):diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 89169fd7c..c7d955bc7 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -673,8 +673,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):name = "testspider"def parse(self, response):- for item in items:- yield item+ yield from itemsdata = yield self.run_and_export(TestSpider, settings)return data@@ -2696,8 +2695,7 @@ class BatchDeliveriesTest(FeedExportTestBase):name = "testspider"def parse(self, response):- for item in items:- yield item+ yield from itemswith MockServer() as server:TestSpider.start_urls = [server.url("/")]diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 00da3d485..9ce40f921 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -317,8 +317,7 @@ class CrawlSpiderTest(SpiderTest):rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)def dummy_process_links(self, links):- for link in links:- yield link+ yield from linksspider = _CrawlSpider()output = list(spider._requests_to_follow(response))diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex d167adbb7..38ca8d950 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -170,8 +170,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):class ProcessSpiderOutputSimpleMiddleware:def process_spider_output(self, response, result, spider):- for r in result:- yield r+ yield from resultclass ProcessSpiderOutputAsyncGenMiddleware:@@ -182,8 +181,7 @@ class ProcessSpiderOutputAsyncGenMiddleware:class ProcessSpiderOutputUniversalMiddleware:def process_spider_output(self, response, result, spider):- for r in result:- yield r+ yield from resultasync def process_spider_output_async(self, response, result, spider):async for r in result:@@ -324,8 +322,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):class ProcessStartRequestsSimpleMiddleware:def process_start_requests(self, start_requests, spider):- for r in start_requests:- yield r+ yield from start_requestsclass ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex bb0ebc2a4..a7d54b565 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -107,8 +107,7 @@ class DeferUtilsTest(unittest.TestCase):class IterErrbackTest(unittest.TestCase):def test_iter_errback_good(self):def itergood():- for x in range(10):- yield x+ yield from range(10)errors = []out = list(iter_errback(itergood(), errors.append))