Scrapy, a fast high-level web crawling & scraping framework for Python.
Add main infrastructure for async callbacks.
Andrey Rakhmatullin2152a2a50898
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex 763e0cdf6..961606f29 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -3,6 +3,7 @@ Spider Middleware managerSee documentation in docs/topics/spider-middleware.rst"""+import inspectfrom itertools import islicefrom twisted.python.failure import Failure@@ -11,11 +12,11 @@ from scrapy.exceptions import _InvalidOutputfrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.conf import build_component_listfrom scrapy.utils.defer import mustbe_deferred-from scrapy.utils.python import MutableChain+from scrapy.utils.python import MutableAsyncChain, MutableChaindef _isiterable(possible_iterator):- return hasattr(possible_iterator, '__iter__')+ return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__')def _fname(f):@@ -58,15 +59,31 @@ class SpiderMiddlewareManager(MiddlewareManager):return scrape_func(response, request, spider)def _evaluate_iterable(iterable, exception_processor_index, recover_to):- try:- for r in iterable:- yield r- except Exception as ex:+ def _process_exception(ex):exception_result = process_spider_exception(Failure(ex), exception_processor_index)if isinstance(exception_result, Failure):raiserecover_to.extend(exception_result)+ def _evaluate_normal_iterable(iterable):+ try:+ for r in iterable:+ yield r+ except Exception as ex:+ _process_exception(ex)++ async def _evaluate_async_iterable(iterable):+ try:+ async for r in iterable:+ yield r+ except Exception as ex:+ _process_exception(ex)++ if inspect.isasyncgen(iterable):+ return _evaluate_async_iterable(iterable)+ else:+ return _evaluate_normal_iterable(iterable)+def process_spider_exception(_failure, start_index=0):exception = _failure.value# don't handle _InvalidOutput exception@@ -92,7 +109,11 @@ class SpiderMiddlewareManager(MiddlewareManager):def process_spider_output(result, start_index=0):# items in this iterable do not need to go through the process_spider_output# chain, they went through it already from the process_spider_exception method- recovered = MutableChain()+ if inspect.isasyncgen(result):+ iter_class = MutableAsyncChain+ else:+ iter_class = MutableChain+ recovered = iter_class()method_list = islice(self.methods['process_spider_output'], start_index, None)for method_index, method in enumerate(method_list, start=start_index):@@ -113,12 +134,16 @@ class SpiderMiddlewareManager(MiddlewareManager):f"iterable, got {type(result)}")raise _InvalidOutput(msg)- return MutableChain(result, recovered)+ return iter_class(result, recovered)def process_callback_output(result):- recovered = MutableChain()+ if inspect.isasyncgen(result):+ iter_class = MutableAsyncChain+ else:+ iter_class = MutableChain+ recovered = iter_class()result = _evaluate_iterable(result, 0, recovered)- return MutableChain(process_spider_output(result), recovered)+ return iter_class(process_spider_output(result), recovered)dfd = mustbe_deferred(process_spider_input, response)dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex bd5f9c8fc..554edc38c 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -290,3 +290,22 @@ def maybeDeferred_coro(f, *args, **kw):return defer.fail(result)else:return defer.succeed(result)+++def deferred_to_future(d):+ """ Wraps a Deferred into a Future. Requires the asyncio reactor.+ """+ return d.asFuture(asyncio.get_event_loop())+++def maybe_deferred_to_future(d):+ """ Converts a Deferred to something that can be awaited in a callback or other user coroutine.++ If the asyncio reactor is installed, coroutines are wrapped into Futures, and only Futures can be+ awaited inside them. Otherwise, coroutines are wrapped into Deferreds and Deferreds can be awaited+ directly inside them.+ """+ if not is_asyncio_reactor_installed():+ return d+ else:+ return deferred_to_future(d)diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex 59fc9202f..d0fd1757d 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -4,7 +4,6 @@ import loggingfrom scrapy.spiders import Spiderfrom scrapy.utils.defer import deferred_from_corofrom scrapy.utils.misc import arg_to_iter-from scrapy.utils.asyncgen import collect_asyncgenlogger = logging.getLogger(__name__)@@ -12,14 +11,13 @@ logger = logging.getLogger(__name__)def iterate_spider_output(result):if inspect.isasyncgen(result):- d = deferred_from_coro(collect_asyncgen(result))- d.addCallback(iterate_spider_output)- return d+ return resultelif inspect.iscoroutine(result):d = deferred_from_coro(result)d.addCallback(iterate_spider_output)return d- return arg_to_iter(result)+ else:+ return arg_to_iter(deferred_from_coro(result))def iter_spider_classes(module):diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 24c38283a..d8fc25094 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -110,3 +110,10 @@ def mock_google_cloud_storage():bucket_mock.blob.return_value = blob_mockreturn (client_mock, bucket_mock, blob_mock)+++def get_web_client_agent_req(url):+ from twisted.internet import reactor+ from twisted.web.client import Agent # imports twisted.internet.reactor+ agent = Agent(reactor)+ return agent.request(b'GET', url.encode('utf-8'))diff --git a/tests/spiders.py b/tests/spiders.pyindex 106392ea6..3e0ec001b 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -14,7 +14,8 @@ from scrapy.item import Itemfrom scrapy.linkextractors import LinkExtractorfrom scrapy.spiders import Spiderfrom scrapy.spiders.crawl import CrawlSpider, Rule-from scrapy.utils.test import get_from_asyncio_queue+from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future+from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_reqclass MockServerSpider(Spider):@@ -148,6 +149,41 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):return reqs+class AsyncDefAsyncioGenExcSpider(SimpleSpider):+ name = 'asyncdef_asyncio_gen_exc'++ async def parse(self, response):+ for i in range(10):+ await asyncio.sleep(0.1)+ yield {'foo': i}+ if i > 5:+ raise ValueError("Stopping the processing")+++class AsyncDefDeferredDirectSpider(SimpleSpider):+ name = 'asyncdef_deferred_direct'++ async def parse(self, response):+ resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200"))+ yield {'code': resp.code}+++class AsyncDefDeferredWrappedSpider(SimpleSpider):+ name = 'asyncdef_deferred_wrapped'++ async def parse(self, response):+ resp = await deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200")))+ yield {'code': resp.code}+++class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):+ name = 'asyncdef_deferred_wrapped'++ async def parse(self, response):+ resp = await maybe_deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200")))+ yield {'code': resp.code}++class AsyncDefAsyncioGenSpider(SimpleSpider):name = 'asyncdef_asyncio_gen'diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 1083c1678..cda52f0d4 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -20,12 +20,16 @@ from scrapy.utils.python import to_unicodefrom tests.mockserver import MockServerfrom tests.spiders import (AsyncDefAsyncioGenComplexSpider,+ AsyncDefAsyncioGenExcSpider,AsyncDefAsyncioGenLoopSpider,AsyncDefAsyncioGenSpider,AsyncDefAsyncioReqsReturnSpider,AsyncDefAsyncioReturnSingleElementSpider,AsyncDefAsyncioReturnSpider,AsyncDefAsyncioSpider,+ AsyncDefDeferredDirectSpider,+ AsyncDefDeferredMaybeWrappedSpider,+ AsyncDefDeferredWrappedSpider,AsyncDefSpider,BrokenStartRequestsSpider,BytesReceivedCallbackSpider,@@ -430,6 +434,18 @@ class CrawlSpiderTestCase(TestCase):for i in range(10):self.assertIn({'foo': i}, items)+ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncgen_parse_exc(self):+ log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider)+ log = str(log)+ self.assertIn("Spider error processing", log)+ self.assertIn("ValueError", log)+ itemcount = stats.get_value('item_scraped_count')+ self.assertEqual(itemcount, 7)+ for i in range(7):+ self.assertIn({'foo': i}, items)+@mark.only_asyncio()@defer.inlineCallbacksdef test_async_def_asyncgen_parse_complex(self):@@ -449,6 +465,23 @@ class CrawlSpiderTestCase(TestCase):for req_id in range(3):self.assertIn(f"Got response 200, req_id {req_id}", str(log))+ @mark.only_not_asyncio()+ @defer.inlineCallbacks+ def test_async_def_deferred_direct(self):+ _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider)+ self.assertEqual(items, [{'code': 200}])++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_deferred_wrapped(self):+ log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider)+ self.assertEqual(items, [{'code': 200}])++ @defer.inlineCallbacks+ def test_async_def_deferred_maybe_wrapped(self):+ _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider)+ self.assertEqual(items, [{'code': 200}])+@defer.inlineCallbacksdef test_response_ssl_certificate_none(self):crawler = self.runner.create_crawler(SingleRequestSpider)diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 029bf8bd6..4d1a7fcb0 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -43,6 +43,23 @@ class RecoverySpider(Spider):raise TabError()+class RecoveryAsyncGenSpider(RecoverySpider):+ name = 'RecoveryAsyncGenSpider'++ async def parse(self, response):+ for r in super().parse(response):+ yield r+++class RecoveryMiddleware:+ def process_spider_exception(self, response, exception, spider):+ spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)+ return [+ {'from': 'process_spider_exception'},+ Request(response.url, meta={'dont_fail': True}, dont_filter=True),+ ]++# ================================================================================# (1) exceptions from a spider middleware's process_spider_input methodclass FailProcessSpiderInputMiddleware:@@ -307,6 +324,16 @@ class TestSpiderMiddleware(TestCase):self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)self.assertIn("'item_scraped_count': 3", str(log))+ @defer.inlineCallbacks+ def test_recovery_asyncgen(self):+ """+ Same as test_recovery but with an async callback.+ """+ log = yield self.crawl_log(RecoveryAsyncGenSpider)+ self.assertIn("Middleware: TabError exception caught", str(log))+ self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)+ self.assertIn("'item_scraped_count': 3", str(log))+@defer.inlineCallbacksdef test_process_spider_input_without_errback(self):"""