Scrapy, a fast high-level web crawling & scraping framework for Python.
Add aiter_errback.
Andrey Rakhmatullinacff1eb49609
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 6db9cc117..2d02c0621 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -124,6 +124,20 @@ def iter_errback(iterable, errback, *a, **kw):errback(failure.Failure(), *a, **kw)+async def aiter_errback(aiterable, errback, *a, **kw):+ """Wraps an async iterable calling an errback if an error is caught while+ iterating it. Similar to scrapy.utils.defer.iter_errback()+ """+ it = aiterable.__aiter__()+ while True:+ try:+ yield await it.__anext__()+ except StopAsyncIteration:+ break+ except Exception:+ errback(failure.Failure(), *a, **kw)++def deferred_from_coro(o):"""Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""if isinstance(o, defer.Deferred):diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex e60242a3b..06d91c574 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -2,12 +2,15 @@ from twisted.trial import unittestfrom twisted.internet import reactor, deferfrom twisted.python.failure import Failure+from scrapy.utils.asyncgen import collect_asyncgenfrom scrapy.utils.defer import (iter_errback,+ aiter_errback,mustbe_deferred,process_chain,process_chain_both,process_parallel,+ deferred_f_from_coro_f,)@@ -117,3 +120,31 @@ class IterErrbackTest(unittest.TestCase):self.assertEqual(out, [0, 1, 2, 3, 4])self.assertEqual(len(errors), 1)self.assertIsInstance(errors[0].value, ZeroDivisionError)+++class AiterErrbackTest(unittest.TestCase):++ @deferred_f_from_coro_f+ async def test_aiter_errback_good(self):+ async def itergood():+ for x in range(10):+ yield x++ errors = []+ out = await collect_asyncgen(aiter_errback(itergood(), errors.append))+ self.assertEqual(out, list(range(10)))+ self.assertFalse(errors)++ @deferred_f_from_coro_f+ async def test_iter_errback_bad(self):+ async def iterbad():+ for x in range(10):+ if x == 5:+ 1 / 0+ yield x++ errors = []+ out = await collect_asyncgen(aiter_errback(iterbad(), errors.append))+ self.assertEqual(out, [0, 1, 2, 3, 4])+ self.assertEqual(len(errors), 1)+ self.assertIsInstance(errors[0].value, ZeroDivisionError)