Scrapy, a fast high-level web crawling & scraping framework for Python.
Support yield in async def callbacks.
Andrey Rakhmatullin7323780c97e6
diff --git a/conftest.py b/conftest.pyindex c0de09909..be5fbabf4 100644--- a/conftest.py+++ b/conftest.py@@ -11,7 +11,9 @@ collect_ignore = [# not a test, but looks like a test"scrapy/utils/testsite.py",# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess- *_py_files("tests/CrawlerProcess")+ *_py_files("tests/CrawlerProcess"),+ # Py36-only parts of respective tests+ *_py_files("tests/py36"),]for line in open('tests/ignores.txt'):diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.pynew file mode 100644index 000000000..c8c24076e--- /dev/null+++ b/scrapy/utils/py36.py@@ -0,0 +1,10 @@+"""+Helpers using Python 3.6+ syntax (ignore SyntaxError on import).+"""+++async def collect_asyncgen(result):+ results = []+ async for x in result:+ results.append(x)+ return resultsdiff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex 72775df5c..4e2a4d1bc 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -4,12 +4,20 @@ import inspectfrom scrapy.spiders import Spiderfrom scrapy.utils.defer import deferred_from_corofrom scrapy.utils.misc import arg_to_iter+try:+ from scrapy.utils.py36 import collect_asyncgen+except SyntaxError:+ collect_asyncgen = Nonelogger = logging.getLogger(__name__)def iterate_spider_output(result):+ if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result):+ d = deferred_from_coro(collect_asyncgen(result))+ d.addCallback(iterate_spider_output)+ return dreturn arg_to_iter(deferred_from_coro(result))diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.pynew file mode 100644index 000000000..74c7daf53--- /dev/null+++ b/tests/py36/_test_crawl.py@@ -0,0 +1,50 @@+import asyncio++from scrapy import Request+from tests.spiders import SimpleSpider+++class AsyncDefAsyncioGenSpider(SimpleSpider):++ name = 'asyncdef_asyncio_gen'++ async def parse(self, response):+ await asyncio.sleep(0.2)+ yield {'foo': 42}+ self.logger.info("Got response %d" % response.status)+++class AsyncDefAsyncioGenLoopSpider(SimpleSpider):++ name = 'asyncdef_asyncio_gen_loop'++ async def parse(self, response):+ for i in range(10):+ await asyncio.sleep(0.1)+ yield {'foo': i}+ self.logger.info("Got response %d" % response.status)+++class AsyncDefAsyncioGenComplexSpider(SimpleSpider):++ name = 'asyncdef_asyncio_gen_complex'+ initial_reqs = 4+ following_reqs = 3+ depth = 2++ def _get_req(self, index):+ return Request(self.mockserver.url("/status?n=200&request=%d" % index),+ meta={'index': index})++ def start_requests(self):+ for i in range(self.initial_reqs):+ yield self._get_req(i)++ async def parse(self, response):+ index = response.meta['index']+ yield {'index': index}+ if index < 10 ** self.depth:+ for new_index in range(10 * index, 10 * index + self.following_reqs):+ yield self._get_req(new_index)+ await asyncio.sleep(0.1)+ yield {'index': index + 5}diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 85005eba4..856068465 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -1,5 +1,6 @@import jsonimport logging+import sysfrom pytest import markfrom testfixtures import LogCapture@@ -343,3 +344,53 @@ with multiples linesself.assertIn("Got response 200", str(log))self.assertIn({'id': 1}, items)self.assertIn({'id': 2}, items)++ @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")+ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncgen_parse(self):+ from tests.py36._test_crawl import AsyncDefAsyncioGenSpider+ crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider)+ with LogCapture() as log:+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ self.assertIn("Got response 200", str(log))+ itemcount = crawler.stats.get_value('item_scraped_count')+ self.assertEqual(itemcount, 1)++ @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")+ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncgen_parse_loop(self):+ items = []++ def _on_item_scraped(item):+ items.append(item)++ from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider+ crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider)+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)+ with LogCapture() as log:+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ self.assertIn("Got response 200", str(log))+ itemcount = crawler.stats.get_value('item_scraped_count')+ self.assertEqual(itemcount, 10)+ for i in range(10):+ self.assertIn({'foo': i}, items)++ @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")+ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncgen_parse_complex(self):+ items = []++ def _on_item_scraped(item):+ items.append(item)++ from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider+ crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider)+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)+ yield crawler.crawl(mockserver=self.mockserver)+ itemcount = crawler.stats.get_value('item_scraped_count')+ self.assertEqual(itemcount, 80)+ for i in [0, 3, 21, 22, 207, 311]: # some random items+ self.assertIn({'index': i}, items)