Scrapy, a fast high-level web crawling & scraping framework for Python.
Add as_async_generator.
Andrey Rakhmatullin76abcedaf4f3
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.pyindex 7f697af5f..db2173f85 100644--- a/scrapy/utils/asyncgen.py+++ b/scrapy/utils/asyncgen.py@@ -1,5 +1,17 @@+import collections++async def collect_asyncgen(result):results = []async for x in result:results.append(x)return results+++async def as_async_generator(it):+ if isinstance(it, collections.abc.AsyncIterator):+ async for r in it:+ yield r+ else:+ for r in it:+ yield rdiff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.pynew file mode 100644index 000000000..9ae66c57c--- /dev/null+++ b/tests/test_utils_asyncgen.py@@ -0,0 +1,20 @@+from twisted.trial import unittest++from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen+from scrapy.utils.defer import deferred_f_from_coro_f+++class AsyncgenUtilsTest(unittest.TestCase):+ @deferred_f_from_coro_f+ async def test_as_async_generator(self):+ ag = as_async_generator(range(42))+ results = []+ async for i in ag:+ results.append(i)+ self.assertEqual(results, list(range(42)))++ @deferred_f_from_coro_f+ async def test_collect_asyncgen(self):+ ag = as_async_generator(range(42))+ results = await collect_asyncgen(ag)+ self.assertEqual(results, list(range(42)))