Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add more tests for _process_iterable_universal.

Andrey Rakhmatullin6803779eb7e4
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 39c94ad8a..a79552f76 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -57,6 +57,6 @@ def _process_iterable_universal(process_async):
if hasattr(iterable, '__iter__'):
# convert process_async to process_sync
return process_sync(iterable, *args, **kwargs)
- raise ValueError(f"Wrong iterable type {type(iterable)}")
+ raise TypeError(f"Wrong iterable type {type(iterable)}")
return process
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index 2f4181d3d..41993a934 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -2,6 +2,7 @@ from twisted.trial import unittest
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal
from scrapy.utils.defer import deferred_f_from_coro_f
+from scrapy.utils.test import get_web_client_agent_req
class AsyncgenUtilsTest(unittest.TestCase):
@@ -26,6 +27,13 @@ async def process_iterable(iterable):
yield i * 2
+@_process_iterable_universal
+async def process_iterable_awaiting(iterable):
+ async for i in iterable:
+ yield i * 2
+ await get_web_client_agent_req('http://example.com')
+
+
class ProcessIterableUniversalTest(unittest.TestCase):
def test_normal(self):
@@ -38,3 +46,22 @@ class ProcessIterableUniversalTest(unittest.TestCase):
iterable = as_async_generator([1, 2, 3])
results = await collect_asyncgen(process_iterable(iterable))
self.assertEqual(results, [2, 4, 6])
+
+ @deferred_f_from_coro_f
+ async def test_blocking(self):
+ iterable = [1, 2, 3]
+ with self.assertRaisesRegex(RuntimeError, "Synchronously-called function"):
+ list(process_iterable_awaiting(iterable))
+
+ def test_invalid_iterable(self):
+ with self.assertRaisesRegex(TypeError, "Wrong iterable type"):
+ process_iterable(None)
+
+ @deferred_f_from_coro_f
+ async def test_invalid_process(self):
+ @_process_iterable_universal
+ def process_iterable_invalid(iterable):
+ pass
+
+ with self.assertRaisesRegex(ValueError, "process_async returned wrong type"):
+ list(process_iterable_invalid([]))