Scrapy, a fast high-level web crawling & scraping framework for Python.
Return single element from coroutine callback (#4609)
Eugenio Lacuesta91e505edfe81
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex 1b8a82829..7e7a50c88 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -1,5 +1,5 @@-import loggingimport inspect+import loggingfrom scrapy.spiders import Spiderfrom scrapy.utils.defer import deferred_from_coro@@ -18,7 +18,11 @@ def iterate_spider_output(result):d = deferred_from_coro(collect_asyncgen(result))d.addCallback(iterate_spider_output)return d- return arg_to_iter(deferred_from_coro(result))+ elif inspect.iscoroutine(result):+ d = deferred_from_coro(result)+ d.addCallback(iterate_spider_output)+ return d+ return arg_to_iter(result)def iter_spider_classes(module):diff --git a/tests/spiders.py b/tests/spiders.pyindex 05078cc04..a360d8206 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -119,6 +119,17 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):return [{'id': 1}, {'id': 2}]+class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):++ name = "asyncdef_asyncio_return_single_element"++ async def parse(self, response):+ await asyncio.sleep(0.1)+ status = await get_from_asyncio_queue(response.status)+ self.logger.info("Got response %d" % status)+ return {"foo": 42}++class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):name = 'asyncdef_asyncio_reqs_return'diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 0115b8fb9..df920f2a2 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -21,6 +21,7 @@ from scrapy.utils.python import to_unicodefrom tests.mockserver import MockServerfrom tests.spiders import (AsyncDefAsyncioReqsReturnSpider,+ AsyncDefAsyncioReturnSingleElementSpider,AsyncDefAsyncioReturnSpider,AsyncDefAsyncioSpider,AsyncDefSpider,@@ -364,6 +365,21 @@ with multiples linesself.assertIn({'id': 1}, items)self.assertIn({'id': 2}, items)+ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncio_parse_items_single_element(self):+ items = []++ def _on_item_scraped(item):+ items.append(item)++ crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider)+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)+ with LogCapture() as log:+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ self.assertIn("Got response 200", str(log))+ self.assertIn({"foo": 42}, items)+@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")@mark.only_asyncio()@defer.inlineCallbacks