Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add more tests for spider middlewares.

Andrey Rakhmatullinf422861ef49e
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 78e926adc..2584dec21 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,11 +1,15 @@
+import collections.abc
from unittest import mock
+from twisted.internet import defer
from twisted.trial.unittest import TestCase
from twisted.python.failure import Failure
from scrapy.spiders import Spider
from scrapy.http import Request, Response
from scrapy.exceptions import _InvalidOutput
+from scrapy.utils.asyncgen import _process_iterable_universal, as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.test import get_crawler
from scrapy.core.spidermw import SpiderMiddlewareManager
@@ -101,3 +105,187 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
result = self._scrape_response()
self.assertIsInstance(result, Failure)
self.assertIsInstance(result.value, ZeroDivisionError)
+
+
+class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
+ """ Helpers for testing sync, async and mixed middlewares.
+
+ Should work for process_spider_output and, when it's supported, process_start_requests.
+ """
+
+ RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects
+
+ @defer.inlineCallbacks
+ def _get_middleware_result(self, *mw_classes):
+ for mw_cls in mw_classes:
+ self.mwman._add_middleware(mw_cls())
+ result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
+ return result
+
+ def assertAsyncGeneratorNotIterable(self, o):
+ with self.assertRaisesRegex(TypeError,
+ "'(async_generator|MutableAsyncChain)' object is not iterable"):
+ list(o)
+
+ @defer.inlineCallbacks
+ def _test_simple_base(self, *mw_classes):
+ result = yield self._get_middleware_result(*mw_classes)
+ self.assertIsInstance(result, collections.abc.Iterable)
+ result_list = list(result)
+ self.assertEqual(len(result_list), self.RESULT_COUNT)
+ self.assertIsInstance(result_list[0], self.ITEM_TYPE)
+
+ @defer.inlineCallbacks
+ def _test_asyncgen_base(self, *mw_classes):
+ result = yield self._get_middleware_result(*mw_classes)
+ self.assertIsInstance(result, collections.abc.AsyncIterator)
+ result_list = yield deferred_from_coro(collect_asyncgen(result))
+ self.assertEqual(len(result_list), self.RESULT_COUNT)
+ self.assertIsInstance(result_list[0], self.ITEM_TYPE)
+
+ @defer.inlineCallbacks
+ def _test_asyncgen_fail(self, *mw_classes):
+ result = yield self._get_middleware_result(*mw_classes)
+ self.assertIsInstance(result, collections.abc.Iterable)
+ self.assertAsyncGeneratorNotIterable(result)
+
+
+class ProcessSpiderOutputSimpleMiddleware:
+ def process_spider_output(self, response, result, spider):
+ for r in result:
+ yield r
+
+
+class ProcessSpiderOutputAsyncGenMiddleware:
+ async def process_spider_output(self, response, result, spider):
+ async for r in as_async_generator(result):
+ yield r
+
+
+class ProcessSpiderOutputUniversalMiddleware:
+ def process_spider_output(self, response, result, spider):
+ @_process_iterable_universal
+ async def process(result):
+ async for r in result:
+ yield r
+ return process(result)
+
+
+class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
+ """ process_spider_output tests for simple callbacks"""
+
+ ITEM_TYPE = dict
+ MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
+ MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
+ MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
+
+ def _scrape_func(self, *args, **kwargs):
+ yield {'foo': 1}
+ yield {'foo': 2}
+ yield {'foo': 3}
+
+ def test_simple(self):
+ """ Simple mw """
+ return self._test_simple_base(self.MW_SIMPLE)
+
+ def test_asyncgen(self):
+ """ Asyncgen mw """
+ return self._test_asyncgen_base(self.MW_ASYNCGEN)
+
+ def test_simple_asyncgen(self):
+ """ Simple mw -> asyncgen mw """
+ return self._test_asyncgen_base(self.MW_ASYNCGEN,
+ self.MW_SIMPLE)
+
+ def test_asyncgen_simple(self):
+ """ Asyncgen mw -> simple mw; cannot work """
+ return self._test_asyncgen_fail(self.MW_SIMPLE,
+ self.MW_ASYNCGEN)
+
+ def test_universal(self):
+ """ Universal mw """
+ return self._test_simple_base(self.MW_UNIVERSAL)
+
+ def test_universal_simple(self):
+ """ Universal mw -> simple mw """
+ return self._test_simple_base(self.MW_SIMPLE,
+ self.MW_UNIVERSAL)
+
+ def test_simple_universal(self):
+ """ Simple mw -> universal mw """
+ return self._test_simple_base(self.MW_UNIVERSAL,
+ self.MW_SIMPLE)
+
+ def test_universal_asyncgen(self):
+ """ Universal mw -> asyncgen mw """
+ return self._test_asyncgen_base(self.MW_ASYNCGEN,
+ self.MW_UNIVERSAL)
+
+ def test_asyncgen_universal(self):
+ """ Asyncgen mw -> universal mw """
+ return self._test_asyncgen_base(self.MW_UNIVERSAL,
+ self.MW_ASYNCGEN)
+
+
+class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
+ """ process_spider_output tests for async generator callbacks """
+
+ async def _scrape_func(self, *args, **kwargs):
+ for item in super()._scrape_func():
+ yield item
+
+ def test_simple(self):
+ """ Simple mw; cannot work """
+ return self._test_asyncgen_fail(self.MW_SIMPLE)
+
+ @defer.inlineCallbacks
+ def test_simple_asyncgen(self):
+ """ Simple mw -> asyncgen mw; cannot work """
+ result = yield self._get_middleware_result(
+ self.MW_ASYNCGEN,
+ self.MW_SIMPLE)
+ self.assertIsInstance(result, collections.abc.AsyncIterable)
+ self.assertAsyncGeneratorNotIterable(result)
+
+ def test_universal(self):
+ """ Universal mw """
+ return self._test_asyncgen_base(self.MW_UNIVERSAL)
+
+ def test_universal_simple(self):
+ """ Universal mw -> simple mw; cannot work """
+ return self._test_asyncgen_fail(self.MW_SIMPLE,
+ self.MW_UNIVERSAL)
+
+ def test_simple_universal(self):
+ """ Simple mw -> universal mw; cannot work """
+ return self._test_asyncgen_fail(self.MW_UNIVERSAL,
+ self.MW_SIMPLE)
+
+
+class ProcessStartRequestsSimpleMiddleware:
+ def process_start_requests(self, start_requests, spider):
+ for r in start_requests:
+ yield r
+
+
+class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
+ """ process_start_requests tests for simple start_requests"""
+
+ ITEM_TYPE = Request
+ MW_SIMPLE = ProcessStartRequestsSimpleMiddleware
+
+ def _start_requests(self):
+ for i in range(3):
+ yield Request(f'https://example.com/{i}', dont_filter=True)
+
+ @defer.inlineCallbacks
+ def _get_middleware_result(self, *mw_classes):
+ for mw_cls in mw_classes:
+ self.mwman._add_middleware(mw_cls())
+ start_requests = iter(self._start_requests())
+ results = yield self.mwman.process_start_requests(start_requests, self.spider)
+ return results
+
+ def test_simple(self):
+ """ Simple mw """
+ self._test_simple_base(self.MW_SIMPLE)