Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add as_async_generator.

Andrey Rakhmatullin76abcedaf4f3
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 7f697af5f..db2173f85 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,5 +1,17 @@
+import collections
+
+
async def collect_asyncgen(result):
results = []
async for x in result:
results.append(x)
return results
+
+
+async def as_async_generator(it):
+ if isinstance(it, collections.abc.AsyncIterator):
+ async for r in it:
+ yield r
+ else:
+ for r in it:
+ yield r
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
new file mode 100644
index 000000000..9ae66c57c
--- /dev/null
+++ b/tests/test_utils_asyncgen.py
@@ -0,0 +1,20 @@
+from twisted.trial import unittest
+
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_f_from_coro_f
+
+
+class AsyncgenUtilsTest(unittest.TestCase):
+ @deferred_f_from_coro_f
+ async def test_as_async_generator(self):
+ ag = as_async_generator(range(42))
+ results = []
+ async for i in ag:
+ results.append(i)
+ self.assertEqual(results, list(range(42)))
+
+ @deferred_f_from_coro_f
+ async def test_collect_asyncgen(self):
+ ag = as_async_generator(range(42))
+ results = await collect_asyncgen(ag)
+ self.assertEqual(results, list(range(42)))