Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support yield in async def callbacks.

Andrey Rakhmatullin7323780c97e6
diff --git a/conftest.py b/conftest.py
index c0de09909..be5fbabf4 100644
--- a/conftest.py
+++ b/conftest.py
@@ -11,7 +11,9 @@ collect_ignore = [
# not a test, but looks like a test
"scrapy/utils/testsite.py",
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
- *_py_files("tests/CrawlerProcess")
+ *_py_files("tests/CrawlerProcess"),
+ # Py36-only parts of respective tests
+ *_py_files("tests/py36"),
]
for line in open('tests/ignores.txt'):
diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py
new file mode 100644
index 000000000..c8c24076e
--- /dev/null
+++ b/scrapy/utils/py36.py
@@ -0,0 +1,10 @@
+"""
+Helpers using Python 3.6+ syntax (ignore SyntaxError on import).
+"""
+
+
+async def collect_asyncgen(result):
+ results = []
+ async for x in result:
+ results.append(x)
+ return results
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 72775df5c..4e2a4d1bc 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -4,12 +4,20 @@ import inspect
from scrapy.spiders import Spider
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.misc import arg_to_iter
+try:
+ from scrapy.utils.py36 import collect_asyncgen
+except SyntaxError:
+ collect_asyncgen = None
logger = logging.getLogger(__name__)
def iterate_spider_output(result):
+ if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result):
+ d = deferred_from_coro(collect_asyncgen(result))
+ d.addCallback(iterate_spider_output)
+ return d
return arg_to_iter(deferred_from_coro(result))
diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py
new file mode 100644
index 000000000..74c7daf53
--- /dev/null
+++ b/tests/py36/_test_crawl.py
@@ -0,0 +1,50 @@
+import asyncio
+
+from scrapy import Request
+from tests.spiders import SimpleSpider
+
+
+class AsyncDefAsyncioGenSpider(SimpleSpider):
+
+ name = 'asyncdef_asyncio_gen'
+
+ async def parse(self, response):
+ await asyncio.sleep(0.2)
+ yield {'foo': 42}
+ self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
+
+ name = 'asyncdef_asyncio_gen_loop'
+
+ async def parse(self, response):
+ for i in range(10):
+ await asyncio.sleep(0.1)
+ yield {'foo': i}
+ self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
+
+ name = 'asyncdef_asyncio_gen_complex'
+ initial_reqs = 4
+ following_reqs = 3
+ depth = 2
+
+ def _get_req(self, index):
+ return Request(self.mockserver.url("/status?n=200&request=%d" % index),
+ meta={'index': index})
+
+ def start_requests(self):
+ for i in range(self.initial_reqs):
+ yield self._get_req(i)
+
+ async def parse(self, response):
+ index = response.meta['index']
+ yield {'index': index}
+ if index < 10 ** self.depth:
+ for new_index in range(10 * index, 10 * index + self.following_reqs):
+ yield self._get_req(new_index)
+ await asyncio.sleep(0.1)
+ yield {'index': index + 5}
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 85005eba4..856068465 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,5 +1,6 @@
import json
import logging
+import sys
from pytest import mark
from testfixtures import LogCapture
@@ -343,3 +344,53 @@ with multiples lines
self.assertIn("Got response 200", str(log))
self.assertIn({'id': 1}, items)
self.assertIn({'id': 2}, items)
+
+ @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncgen_parse(self):
+ from tests.py36._test_crawl import AsyncDefAsyncioGenSpider
+ crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider)
+ with LogCapture() as log:
+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ self.assertIn("Got response 200", str(log))
+ itemcount = crawler.stats.get_value('item_scraped_count')
+ self.assertEqual(itemcount, 1)
+
+ @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncgen_parse_loop(self):
+ items = []
+
+ def _on_item_scraped(item):
+ items.append(item)
+
+ from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider
+ crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider)
+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+ with LogCapture() as log:
+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ self.assertIn("Got response 200", str(log))
+ itemcount = crawler.stats.get_value('item_scraped_count')
+ self.assertEqual(itemcount, 10)
+ for i in range(10):
+ self.assertIn({'foo': i}, items)
+
+ @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncgen_parse_complex(self):
+ items = []
+
+ def _on_item_scraped(item):
+ items.append(item)
+
+ from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider
+ crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider)
+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+ yield crawler.crawl(mockserver=self.mockserver)
+ itemcount = crawler.stats.get_value('item_scraped_count')
+ self.assertEqual(itemcount, 80)
+ for i in [0, 3, 21, 22, 207, 311]: # some random items
+ self.assertIn({'index': i}, items)