Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

refactor(yield): use `yield from` syntax

Rotzbua0d445a3224ec
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index dcf1a6dbc..031a0be36 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -103,8 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
) -> Union[Generator, AsyncGenerator]:
def process_sync(iterable: Iterable) -> Generator:
try:
- for r in iterable:
- yield r
+ yield from iterable
except Exception as ex:
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 31e845716..ebb4f5984 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -131,8 +131,7 @@ class CrawlSpider(Spider):
def _handle_failure(self, failure, errback):
if errback:
results = errback(failure) or ()
- for request_or_item in iterate_spider_output(results):
- yield request_or_item
+ yield from iterate_spider_output(results)
def _compile_rules(self):
self._rules = []
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 6afadc577..47827e442 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -58,8 +58,7 @@ class XMLFeedSpider(Spider):
for selector in nodes:
ret = iterate_spider_output(self.parse_node(response, selector))
- for result_item in self.process_results(response, ret):
- yield result_item
+ yield from self.process_results(response, ret)
def _parse(self, response, **kwargs):
if not hasattr(self, "parse_node"):
@@ -133,8 +132,7 @@ class CSVFeedSpider(Spider):
response, self.delimiter, self.headers, quotechar=self.quotechar
):
ret = iterate_spider_output(self.parse_row(response, row))
- for result_item in self.process_results(response, ret):
- yield result_item
+ yield from self.process_results(response, ret)
def _parse(self, response, **kwargs):
if not hasattr(self, "parse_row"):
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index aaf75a519..974665fe0 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -33,8 +33,7 @@ class SitemapSpider(Spider):
attributes, for example, you can filter locs with lastmod greater
than a given date (see docs).
"""
- for entry in entries:
- yield entry
+ yield from entries
def _parse_sitemap(self, response):
if response.url.endswith("/robots.txt"):
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 0b5dc324f..68ca96b69 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -57,8 +57,7 @@ def iflatten(x: Iterable) -> Iterable:
Similar to ``.flatten()``, but returns iterator instead"""
for el in x:
if is_listlike(el):
- for el_ in iflatten(el):
- yield el_
+ yield from iflatten(el)
else:
yield el
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 24fcbd85e..cea1bc727 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -44,8 +44,7 @@ def _serialize_headers(
for header in headers:
if header in request.headers:
yield header
- for value in request.headers.getlist(header):
- yield value
+ yield from request.headers.getlist(header)
def request_fingerprint(
diff --git a/tests/spiders.py b/tests/spiders.py
index f29dea2a1..3df153a12 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -301,8 +301,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
def parse(self, response):
self.seedsseen.append(response.meta.get("seed"))
- for req in super().parse(response):
- yield req
+ yield from super().parse(response)
class SingleRequestSpider(MetaSpider):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 89169fd7c..c7d955bc7 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -673,8 +673,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
name = "testspider"
def parse(self, response):
- for item in items:
- yield item
+ yield from items
data = yield self.run_and_export(TestSpider, settings)
return data
@@ -2696,8 +2695,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
name = "testspider"
def parse(self, response):
- for item in items:
- yield item
+ yield from items
with MockServer() as server:
TestSpider.start_urls = [server.url("/")]
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 00da3d485..9ce40f921 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -317,8 +317,7 @@ class CrawlSpiderTest(SpiderTest):
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
def dummy_process_links(self, links):
- for link in links:
- yield link
+ yield from links
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index d167adbb7..38ca8d950 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -170,8 +170,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
class ProcessSpiderOutputSimpleMiddleware:
def process_spider_output(self, response, result, spider):
- for r in result:
- yield r
+ yield from result
class ProcessSpiderOutputAsyncGenMiddleware:
@@ -182,8 +181,7 @@ class ProcessSpiderOutputAsyncGenMiddleware:
class ProcessSpiderOutputUniversalMiddleware:
def process_spider_output(self, response, result, spider):
- for r in result:
- yield r
+ yield from result
async def process_spider_output_async(self, response, result, spider):
async for r in result:
@@ -324,8 +322,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
class ProcessStartRequestsSimpleMiddleware:
def process_start_requests(self, start_requests, spider):
- for r in start_requests:
- yield r
+ yield from start_requests
class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index bb0ebc2a4..a7d54b565 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -107,8 +107,7 @@ class DeferUtilsTest(unittest.TestCase):
class IterErrbackTest(unittest.TestCase):
def test_iter_errback_good(self):
def itergood():
- for x in range(10):
- yield x
+ yield from range(10)
errors = []
out = list(iter_errback(itergood(), errors.append))