Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Return single element from coroutine callback (#4609)

Eugenio Lacuesta91e505edfe81
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 1b8a82829..7e7a50c88 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,5 +1,5 @@
-import logging
import inspect
+import logging
from scrapy.spiders import Spider
from scrapy.utils.defer import deferred_from_coro
@@ -18,7 +18,11 @@ def iterate_spider_output(result):
d = deferred_from_coro(collect_asyncgen(result))
d.addCallback(iterate_spider_output)
return d
- return arg_to_iter(deferred_from_coro(result))
+ elif inspect.iscoroutine(result):
+ d = deferred_from_coro(result)
+ d.addCallback(iterate_spider_output)
+ return d
+ return arg_to_iter(result)
def iter_spider_classes(module):
diff --git a/tests/spiders.py b/tests/spiders.py
index 05078cc04..a360d8206 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -119,6 +119,17 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
return [{'id': 1}, {'id': 2}]
+class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
+
+ name = "asyncdef_asyncio_return_single_element"
+
+ async def parse(self, response):
+ await asyncio.sleep(0.1)
+ status = await get_from_asyncio_queue(response.status)
+ self.logger.info("Got response %d" % status)
+ return {"foo": 42}
+
+
class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
name = 'asyncdef_asyncio_reqs_return'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 0115b8fb9..df920f2a2 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -21,6 +21,7 @@ from scrapy.utils.python import to_unicode
from tests.mockserver import MockServer
from tests.spiders import (
AsyncDefAsyncioReqsReturnSpider,
+ AsyncDefAsyncioReturnSingleElementSpider,
AsyncDefAsyncioReturnSpider,
AsyncDefAsyncioSpider,
AsyncDefSpider,
@@ -364,6 +365,21 @@ with multiples lines
self.assertIn({'id': 1}, items)
self.assertIn({'id': 2}, items)
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncio_parse_items_single_element(self):
+ items = []
+
+ def _on_item_scraped(item):
+ items.append(item)
+
+ crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider)
+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+ with LogCapture() as log:
+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ self.assertIn("Got response 200", str(log))
+ self.assertIn({"foo": 42}, items)
+
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
@mark.only_asyncio()
@defer.inlineCallbacks