Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add async def support to pipelines.

Andrey Rakhmatullin1f9cef787d3c
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index aa1bfb77f..1a45e00a2 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -6,6 +6,8 @@ See documentation in docs/item-pipeline.rst
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
+from scrapy.utils.defer import deferred_f_from_coro_f
+
class ItemPipelineManager(MiddlewareManager):
@@ -19,7 +21,7 @@ class ItemPipelineManager(MiddlewareManager):
def _add_middleware(self, pipe):
super(ItemPipelineManager, self)._add_middleware(pipe)
if hasattr(pipe, 'process_item'):
- self.methods['process_item'].append(pipe.process_item)
+ self.methods['process_item'].append(deferred_f_from_coro_f(pipe.process_item))
def process_item(self, item, spider):
return self._process_chain('process_item', item, spider)
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index bc53f5427..cfe4471d7 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -26,6 +26,13 @@ class DeferredPipeline:
return d
+class AsyncDefPipeline:
+ async def process_item(self, item, spider):
+ await defer.succeed(42)
+ item['pipeline_passed'] = True
+ return item
+
+
class ItemSpider(Spider):
name = 'itemspider'
@@ -69,3 +76,9 @@ class PipelineTestCase(unittest.TestCase):
crawler = self._create_crawler(DeferredPipeline)
yield crawler.crawl(mockserver=self.mockserver)
self.assertEqual(len(self.items), 1)
+
+ @defer.inlineCallbacks
+ def test_asyncdef_pipeline(self):
+ crawler = self._create_crawler(AsyncDefPipeline)
+ yield crawler.crawl(mockserver=self.mockserver)
+ self.assertEqual(len(self.items), 1)