Scrapy, a fast high-level web crawling & scraping framework for Python.
Support for async def callbacks.
Andrey Rakhmatullina91a13b4434a
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex 4061d1ea3..72775df5c 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -2,14 +2,15 @@ import loggingimport inspectfrom scrapy.spiders import Spider-from scrapy.utils.misc import arg_to_iter+from scrapy.utils.defer import deferred_from_coro+from scrapy.utils.misc import arg_to_iterlogger = logging.getLogger(__name__)def iterate_spider_output(result):- return arg_to_iter(result)+ return arg_to_iter(deferred_from_coro(result))def iter_spider_classes(module):diff --git a/tests/spiders.py b/tests/spiders.pyindex 39c8da0b6..e4f2d5474 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -1,14 +1,18 @@"""Some spiders used for testing and benchmarking"""+import asyncioimport timefrom urllib.parse import urlencode+from twisted.internet import defer+from scrapy.http import Requestfrom scrapy.item import Itemfrom scrapy.linkextractors import LinkExtractorfrom scrapy.spiders import Spiderfrom scrapy.spiders.crawl import CrawlSpider, Rule+from scrapy.utils.test import get_from_asyncio_queueclass MockServerSpider(Spider):@@ -83,6 +87,25 @@ class SimpleSpider(MetaSpider):self.logger.info("Got response %d" % response.status)+class AsyncDefSpider(SimpleSpider):++ name = 'asyncdef'++ async def parse(self, response):+ await defer.succeed(42)+ self.logger.info("Got response %d" % response.status)+++class AsyncDefAsyncioSpider(SimpleSpider):++ name = 'asyncdef_asyncio'++ async def parse(self, response):+ await asyncio.sleep(0.2)+ status = await get_from_asyncio_queue(response.status)+ self.logger.info("Got response %d" % status)++class ItemSpider(FollowAllSpider):name = 'item'diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex f433fcea6..99b887ff6 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -1,6 +1,7 @@import jsonimport logging+from pytest import markfrom testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.trial.unittest import TestCase@@ -10,7 +11,8 @@ from scrapy.http import Requestfrom scrapy.utils.python import to_unicodefrom tests.mockserver import MockServerfrom tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,- SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback)+ SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,+ AsyncDefSpider, AsyncDefAsyncioSpider)class CrawlTestCase(TestCase):@@ -308,3 +310,19 @@ with multiples linesself.assertIn("[callback] status 201", str(log))self.assertIn("[errback] status 404", str(log))self.assertIn("[errback] status 500", str(log))++ @defer.inlineCallbacks+ def test_async_def_parse(self):+ self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ with LogCapture() as log:+ yield self.runner.join()+ self.assertIn("Got response 200", str(log))++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncio_parse(self):+ runner = CrawlerRunner({"ASYNCIO_REACTOR": True})+ runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ with LogCapture() as log:+ yield runner.join()+ self.assertIn("Got response 200", str(log))