Scrapy, a fast high-level web crawling & scraping framework for Python.
Drop process_iterable_helper, add _process_iterable_universal.
Andrey Rakhmatullinc51ec1ae1cef
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex d3d7e5f8c..33e215971 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -10,6 +10,7 @@ from twisted.python.failure import Failurefrom scrapy.exceptions import _InvalidOutputfrom scrapy.middleware import MiddlewareManager+from scrapy.utils.asyncgen import _process_iterable_universalfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import mustbe_deferredfrom scrapy.utils.python import MutableAsyncChain, MutableChain@@ -57,31 +58,18 @@ class SpiderMiddlewareManager(MiddlewareManager):return scrape_func(response, request, spider)def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to):- def _process_exception(ex):- exception_result = self._process_spider_exception(response, spider, Failure(ex),- exception_processor_index)- if isinstance(exception_result, Failure):- raise # pylint: disable=E0704- recover_to.extend(exception_result)-- def _evaluate_normal_iterable(iterable):- try:- for r in iterable:- yield r- except Exception as ex:- _process_exception(ex)-+ @_process_iterable_universalasync def _evaluate_async_iterable(iterable):try:async for r in iterable:yield rexcept Exception as ex:- _process_exception(ex)-- if inspect.isasyncgen(iterable):- return _evaluate_async_iterable(iterable)- else:- return _evaluate_normal_iterable(iterable)+ exception_result = self._process_spider_exception(response, spider, Failure(ex),+ exception_processor_index)+ if isinstance(exception_result, Failure):+ raise+ recover_to.extend(exception_result)+ return _evaluate_async_iterable(iterable)def _process_spider_exception(self, response, spider, _failure, start_index=0):exception = _failure.valuediff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.pyindex 776a6879a..973404b2b 100644--- a/scrapy/spidermiddlewares/depth.py+++ b/scrapy/spidermiddlewares/depth.py@@ -7,6 +7,7 @@ See documentation in docs/topics/spider-middleware.rstimport loggingfrom scrapy.http import Request+from scrapy.utils.asyncgen import _process_iterable_universallogger = logging.getLogger(__name__)@@ -49,10 +50,15 @@ class DepthMiddleware:spider=spider)return True- # base case (depth=0)- if 'depth' not in response.meta:- response.meta['depth'] = 0- if self.verbose_stats:- self.stats.inc_value('request_depth_count/0', spider=spider)+ @_process_iterable_universal+ async def process(result):+ # base case (depth=0)+ if 'depth' not in response.meta:+ response.meta['depth'] = 0+ if self.verbose_stats:+ self.stats.inc_value('request_depth_count/0', spider=spider)- return (r for r in result or () if _filter(r))+ async for r in result or ():+ if _filter(r):+ yield r+ return process(result)diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.pyindex 6e4efda97..074ec7a4e 100644--- a/scrapy/spidermiddlewares/offsite.py+++ b/scrapy/spidermiddlewares/offsite.py@@ -9,6 +9,7 @@ import warningsfrom scrapy import signalsfrom scrapy.http import Request+from scrapy.utils.asyncgen import _process_iterable_universalfrom scrapy.utils.httpobj import urlparse_cachedlogger = logging.getLogger(__name__)@@ -26,21 +27,24 @@ class OffsiteMiddleware:return odef process_spider_output(self, response, result, spider):- for x in result:- if isinstance(x, Request):- if x.dont_filter or self.should_follow(x, spider):- yield x+ @_process_iterable_universal+ async def process(result):+ async for x in result:+ if isinstance(x, Request):+ if x.dont_filter or self.should_follow(x, spider):+ yield x+ else:+ domain = urlparse_cached(x).hostname+ if domain and domain not in self.domains_seen:+ self.domains_seen.add(domain)+ logger.debug(+ "Filtered offsite request to %(domain)r: %(request)s",+ {'domain': domain, 'request': x}, extra={'spider': spider})+ self.stats.inc_value('offsite/domains', spider=spider)+ self.stats.inc_value('offsite/filtered', spider=spider)else:- domain = urlparse_cached(x).hostname- if domain and domain not in self.domains_seen:- self.domains_seen.add(domain)- logger.debug(- "Filtered offsite request to %(domain)r: %(request)s",- {'domain': domain, 'request': x}, extra={'spider': spider})- self.stats.inc_value('offsite/domains', spider=spider)- self.stats.inc_value('offsite/filtered', spider=spider)- else:- yield x+ yield x+ return process(result)def should_follow(self, request, spider):regex = self.host_regexdiff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex f81041376..8d862d1d0 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -10,6 +10,7 @@ from w3lib.url import safe_url_stringfrom scrapy.http import Request, Responsefrom scrapy.exceptions import NotConfiguredfrom scrapy import signals+from scrapy.utils.asyncgen import _process_iterable_universalfrom scrapy.utils.python import to_unicodefrom scrapy.utils.misc import load_objectfrom scrapy.utils.url import strip_url@@ -337,7 +338,13 @@ class RefererMiddleware:if referrer is not None:r.headers.setdefault('Referer', referrer)return r- return (_set_referer(r) for r in result or ())++ @_process_iterable_universal+ async def process(result):+ async for r in result or ():+ yield _set_referer(r)++ return process(result)def request_scheduled(self, request, spider):# check redirected request to patch "Referer" header if necessarydiff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.pyindex 5be1f80cb..d40d43ff1 100644--- a/scrapy/spidermiddlewares/urllength.py+++ b/scrapy/spidermiddlewares/urllength.py@@ -8,6 +8,7 @@ import loggingfrom scrapy.http import Requestfrom scrapy.exceptions import NotConfigured+from scrapy.utils.asyncgen import _process_iterable_universallogger = logging.getLogger(__name__)@@ -34,4 +35,10 @@ class UrlLengthMiddleware:else:return True- return (r for r in result or () if _filter(r))+ @_process_iterable_universal+ async def process(result):+ async for r in result or ():+ if _filter(r):+ yield r++ return process(result)diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.pyindex db2173f85..39c94ad8a 100644--- a/scrapy/utils/asyncgen.py+++ b/scrapy/utils/asyncgen.py@@ -1,4 +1,6 @@import collections+import functools+import inspectasync def collect_asyncgen(result):@@ -15,3 +17,46 @@ async def as_async_generator(it):else:for r in it:yield r+++# https://stackoverflow.com/a/66170760/113586+def _process_iterable_universal(process_async):+ """ Takes a function that takes an async iterable, args and kwargs. Returns+ a function that takes any iterable, args and kwargs.++ Requires that process_async only awaits on the iterable and synchronous functions,+ so it's better to use this only in the Scrapy code itself.+ """++ # If this stops working, all internal uses can be just replaced with manually-written+ # process_sync functions.++ def process_sync(iterable, *args, **kwargs):+ agen = process_async(as_async_generator(iterable), *args, **kwargs)+ if not inspect.isasyncgen(agen):+ raise ValueError(f"process_async returned wrong type {type(agen)}")+ sent = None+ while True:+ try:+ gen = agen.asend(sent)+ gen.send(None)+ except StopIteration as e:+ sent = yield e.value+ except StopAsyncIteration:+ return+ else:+ gen.throw(RuntimeError,+ f"Synchronously-called function '{process_async.__name__}' has blocked, "+ f"you can't use {_process_iterable_universal.__name__} with it.")++ @functools.wraps(process_async)+ def process(iterable, *args, **kwargs):+ if inspect.isasyncgen(iterable):+ # call process_async directly+ return process_async(iterable, *args, **kwargs)+ if hasattr(iterable, '__iter__'):+ # convert process_async to process_sync+ return process_sync(iterable, *args, **kwargs)+ raise ValueError(f"Wrong iterable type {type(iterable)}")++ return processdiff --git a/scrapy/utils/middlewares.py b/scrapy/utils/middlewares.pydeleted file mode 100644index da28e0ddf..000000000--- a/scrapy/utils/middlewares.py+++ /dev/null@@ -1,35 +0,0 @@-# coding: utf-8-import inspect---def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):- for o in it:- if in_predicate and not in_predicate(o):- continue- if processor is not None:- o = processor(o)- if out_predicate and not out_predicate(o):- continue- yield o---async def process_async_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):- async for o in it:- if in_predicate and not in_predicate(o):- continue- if processor is not None:- o = processor(o)- if out_predicate and not out_predicate(o):- continue- yield o---def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):- """- For each item in the iterable: skips it if in_predicate is False, applies processor,- skips the result if out_predicate is False, else yields it.- """- if inspect.isasyncgen(it):- return process_async_iterable_helper(it, in_predicate, out_predicate, processor)- else:- return process_normal_iterable_helper(it, in_predicate, out_predicate, processor)diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.pyindex 9ae66c57c..2f4181d3d 100644--- a/tests/test_utils_asyncgen.py+++ b/tests/test_utils_asyncgen.py@@ -1,6 +1,6 @@from twisted.trial import unittest-from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universalfrom scrapy.utils.defer import deferred_f_from_coro_f@@ -18,3 +18,23 @@ class AsyncgenUtilsTest(unittest.TestCase):ag = as_async_generator(range(42))results = await collect_asyncgen(ag)self.assertEqual(results, list(range(42)))+++@_process_iterable_universal+async def process_iterable(iterable):+ async for i in iterable:+ yield i * 2+++class ProcessIterableUniversalTest(unittest.TestCase):++ def test_normal(self):+ iterable = iter([1, 2, 3])+ results = list(process_iterable(iterable))+ self.assertEqual(results, [2, 4, 6])++ @deferred_f_from_coro_f+ async def test_async(self):+ iterable = as_async_generator([1, 2, 3])+ results = await collect_asyncgen(process_iterable(iterable))+ self.assertEqual(results, [2, 4, 6])diff --git a/tests/test_utils_middlewares.py b/tests/test_utils_middlewares.pydeleted file mode 100644index d395ba1a9..000000000--- a/tests/test_utils_middlewares.py+++ /dev/null@@ -1,87 +0,0 @@-import collections--from twisted.trial import unittest--from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen-from scrapy.utils.defer import deferred_f_from_coro_f-from scrapy.utils.middlewares import process_iterable_helper---def predicate1(o):- return bool(o % 2)---def predicate2(o):- return o < 10---def processor(o):- return o * 2---class ProcessIterableHelperNormalTest(unittest.TestCase):-- def test_normal_in_predicate(self):- iterable1 = iter([1, 2, 3])- iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)- self.assertIsInstance(iterable2, collections.abc.Iterable)- list2 = list(iterable2)- self.assertEqual(list2, [1, 3])-- def test_normal_out_predicate(self):- iterable1 = iter([1, 2, 10, 3, 15])- iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)- self.assertIsInstance(iterable2, collections.abc.Iterable)- list2 = list(iterable2)- self.assertEqual(list2, [1, 2, 3])-- def test_normal_processor(self):- iterable1 = iter([1, 2, 3])- iterable2 = process_iterable_helper(iterable1, processor=processor)- self.assertIsInstance(iterable2, collections.abc.Iterable)- list2 = list(iterable2)- self.assertEqual(list2, [2, 4, 6])-- def test_normal_combined(self):- iterable1 = iter([1, 2, 10, 3, 6, 18, 5, 15])- iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,- out_predicate=predicate2, processor=processor)- self.assertIsInstance(iterable2, collections.abc.Iterable)- list2 = list(iterable2)- self.assertEqual(list2, [2, 6])---class ProcessIterableHelperAsyncTest(unittest.TestCase):-- @deferred_f_from_coro_f- async def test_async_in_predicate(self):- iterable1 = as_async_generator([1, 2, 3])- iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)- self.assertIsInstance(iterable2, collections.abc.AsyncIterable)- list2 = await collect_asyncgen(iterable2)- self.assertEqual(list2, [1, 3])-- @deferred_f_from_coro_f- async def test_async_out_predicate(self):- iterable1 = as_async_generator([1, 2, 10, 3, 15])- iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)- self.assertIsInstance(iterable2, collections.abc.AsyncIterable)- list2 = await collect_asyncgen(iterable2)- self.assertEqual(list2, [1, 2, 3])-- @deferred_f_from_coro_f- async def test_async_processor(self):- iterable1 = as_async_generator([1, 2, 3])- iterable2 = process_iterable_helper(iterable1, processor=processor)- self.assertIsInstance(iterable2, collections.abc.AsyncIterable)- list2 = await collect_asyncgen(iterable2)- self.assertEqual(list2, [2, 4, 6])-- @deferred_f_from_coro_f- async def test_async_combined(self):- iterable1 = as_async_generator([1, 2, 10, 3, 6, 18, 5, 15])- iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,- out_predicate=predicate2, processor=processor)- self.assertIsInstance(iterable2, collections.abc.AsyncIterable)- list2 = await collect_asyncgen(iterable2)- self.assertEqual(list2, [2, 6])