Scrapy, a fast high-level web crawling & scraping framework for Python.
Add tests for _AsyncCooperatorAdapter.
Andrey Rakhmatullina6034f942b03
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 62f6ff194..543bbee09 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -1,14 +1,18 @@+import random+from pytest import markfrom twisted.trial import unittestfrom twisted.internet import reactor, deferfrom twisted.python.failure import Failure-from scrapy.utils.asyncgen import collect_asyncgen+from scrapy.utils.asyncgen import collect_asyncgen, as_async_generatorfrom scrapy.utils.defer import (aiter_errback,deferred_f_from_coro_f,iter_errback,+ maybe_deferred_to_future,mustbe_deferred,+ parallel_async,process_chain,process_chain_both,process_parallel,@@ -164,3 +168,65 @@ class AsyncDefTestsuiteTest(unittest.TestCase):@deferred_f_from_coro_fasync def test_deferred_f_from_coro_f_xfail(self):raise Exception("This is expected to be raised")+++class AsyncCooperatorTest(unittest.TestCase):+ """ This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.++ parallel_async is called with the results of a callback (so an iterable of items, requests and None,+ with arbitrary delays between values), and it uses Scraper._process_spidermw_output as the callable+ (so a callable that returns a Deferred for an item, which will fire after pipelines process it, and+ None for everything else). The concurrent task count is the CONCURRENT_ITEMS setting.++ We want to test different concurrency values compared to the iterable length.+ We also want to simulate the real usage, with arbitrary delays between getting the values+ from the iterable. We also want to simulate sync and async results from the callable.+ """+ CONCURRENT_ITEMS = 50++ @staticmethod+ def callable(o, results):+ if random.random() < 0.4:+ # simulate async processing+ dfd = defer.Deferred()+ dfd.addCallback(lambda _: results.append(o))+ delay = random.random() / 8+ reactor.callLater(delay, dfd.callback, None)+ return dfd+ else:+ # simulate trivial sync processing+ results.append(o)++ @staticmethod+ def get_async_iterable(length):+ # simulate a simple callback without delays between results+ return as_async_generator(range(length))++ @staticmethod+ async def get_async_iterable_with_delays(length):+ # simulate a callback with delays between some of the results+ for i in range(length):+ if random.random() < 0.1:+ dfd = defer.Deferred()+ delay = random.random() / 20+ reactor.callLater(delay, dfd.callback, None)+ await maybe_deferred_to_future(dfd)+ yield i++ @defer.inlineCallbacks+ def test_simple(self):+ for length in [20, 50, 100]:+ results = []+ ait = self.get_async_iterable(length)+ dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results)+ yield dl+ self.assertEqual(list(range(length)), sorted(results))++ @defer.inlineCallbacks+ def test_delays(self):+ for length in [20, 50, 100]:+ results = []+ ait = self.get_async_iterable_with_delays(length)+ dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results)+ yield dl+ self.assertEqual(list(range(length)), sorted(results))