Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support for async def callbacks.

Andrey Rakhmatullina91a13b4434a
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 4061d1ea3..72775df5c 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -2,14 +2,15 @@ import logging
import inspect
from scrapy.spiders import Spider
-from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.misc import arg_to_iter
logger = logging.getLogger(__name__)
def iterate_spider_output(result):
- return arg_to_iter(result)
+ return arg_to_iter(deferred_from_coro(result))
def iter_spider_classes(module):
diff --git a/tests/spiders.py b/tests/spiders.py
index 39c8da0b6..e4f2d5474 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -1,14 +1,18 @@
"""
Some spiders used for testing and benchmarking
"""
+import asyncio
import time
from urllib.parse import urlencode
+from twisted.internet import defer
+
from scrapy.http import Request
from scrapy.item import Item
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
from scrapy.spiders.crawl import CrawlSpider, Rule
+from scrapy.utils.test import get_from_asyncio_queue
class MockServerSpider(Spider):
@@ -83,6 +87,25 @@ class SimpleSpider(MetaSpider):
self.logger.info("Got response %d" % response.status)
+class AsyncDefSpider(SimpleSpider):
+
+ name = 'asyncdef'
+
+ async def parse(self, response):
+ await defer.succeed(42)
+ self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioSpider(SimpleSpider):
+
+ name = 'asyncdef_asyncio'
+
+ async def parse(self, response):
+ await asyncio.sleep(0.2)
+ status = await get_from_asyncio_queue(response.status)
+ self.logger.info("Got response %d" % status)
+
+
class ItemSpider(FollowAllSpider):
name = 'item'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index f433fcea6..99b887ff6 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,6 +1,7 @@
import json
import logging
+from pytest import mark
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase
@@ -10,7 +11,8 @@ from scrapy.http import Request
from scrapy.utils.python import to_unicode
from tests.mockserver import MockServer
from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
- SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback)
+ SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,
+ AsyncDefSpider, AsyncDefAsyncioSpider)
class CrawlTestCase(TestCase):
@@ -308,3 +310,19 @@ with multiples lines
self.assertIn("[callback] status 201", str(log))
self.assertIn("[errback] status 404", str(log))
self.assertIn("[errback] status 500", str(log))
+
+ @defer.inlineCallbacks
+ def test_async_def_parse(self):
+ self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ with LogCapture() as log:
+ yield self.runner.join()
+ self.assertIn("Got response 200", str(log))
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncio_parse(self):
+ runner = CrawlerRunner({"ASYNCIO_REACTOR": True})
+ runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ with LogCapture() as log:
+ yield runner.join()
+ self.assertIn("Got response 200", str(log))