Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add tests for _AsyncCooperatorAdapter.

Andrey Rakhmatullina6034f942b03
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 62f6ff194..543bbee09 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -1,14 +1,18 @@
+import random
+
from pytest import mark
from twisted.trial import unittest
from twisted.internet import reactor, defer
from twisted.python.failure import Failure
-from scrapy.utils.asyncgen import collect_asyncgen
+from scrapy.utils.asyncgen import collect_asyncgen, as_async_generator
from scrapy.utils.defer import (
aiter_errback,
deferred_f_from_coro_f,
iter_errback,
+ maybe_deferred_to_future,
mustbe_deferred,
+ parallel_async,
process_chain,
process_chain_both,
process_parallel,
@@ -164,3 +168,65 @@ class AsyncDefTestsuiteTest(unittest.TestCase):
@deferred_f_from_coro_f
async def test_deferred_f_from_coro_f_xfail(self):
raise Exception("This is expected to be raised")
+
+
+class AsyncCooperatorTest(unittest.TestCase):
+ """ This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.
+
+ parallel_async is called with the results of a callback (so an iterable of items, requests and None,
+ with arbitrary delays between values), and it uses Scraper._process_spidermw_output as the callable
+ (so a callable that returns a Deferred for an item, which will fire after pipelines process it, and
+ None for everything else). The concurrent task count is the CONCURRENT_ITEMS setting.
+
+ We want to test different concurrency values compared to the iterable length.
+ We also want to simulate the real usage, with arbitrary delays between getting the values
+ from the iterable. We also want to simulate sync and async results from the callable.
+ """
+ CONCURRENT_ITEMS = 50
+
+ @staticmethod
+ def callable(o, results):
+ if random.random() < 0.4:
+ # simulate async processing
+ dfd = defer.Deferred()
+ dfd.addCallback(lambda _: results.append(o))
+ delay = random.random() / 8
+ reactor.callLater(delay, dfd.callback, None)
+ return dfd
+ else:
+ # simulate trivial sync processing
+ results.append(o)
+
+ @staticmethod
+ def get_async_iterable(length):
+ # simulate a simple callback without delays between results
+ return as_async_generator(range(length))
+
+ @staticmethod
+ async def get_async_iterable_with_delays(length):
+ # simulate a callback with delays between some of the results
+ for i in range(length):
+ if random.random() < 0.1:
+ dfd = defer.Deferred()
+ delay = random.random() / 20
+ reactor.callLater(delay, dfd.callback, None)
+ await maybe_deferred_to_future(dfd)
+ yield i
+
+ @defer.inlineCallbacks
+ def test_simple(self):
+ for length in [20, 50, 100]:
+ results = []
+ ait = self.get_async_iterable(length)
+ dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results)
+ yield dl
+ self.assertEqual(list(range(length)), sorted(results))
+
+ @defer.inlineCallbacks
+ def test_delays(self):
+ for length in [20, 50, 100]:
+ results = []
+ ait = self.get_async_iterable_with_delays(length)
+ dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results)
+ yield dl
+ self.assertEqual(list(range(length)), sorted(results))