Scrapy, a fast high-level web crawling & scraping framework for Python.
Add async def support to pipelines.
Andrey Rakhmatullin1f9cef787d3c
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.pyindex aa1bfb77f..1a45e00a2 100644--- a/scrapy/pipelines/__init__.py+++ b/scrapy/pipelines/__init__.py@@ -6,6 +6,8 @@ See documentation in docs/item-pipeline.rstfrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.conf import build_component_list+from scrapy.utils.defer import deferred_f_from_coro_f+class ItemPipelineManager(MiddlewareManager):@@ -19,7 +21,7 @@ class ItemPipelineManager(MiddlewareManager):def _add_middleware(self, pipe):super(ItemPipelineManager, self)._add_middleware(pipe)if hasattr(pipe, 'process_item'):- self.methods['process_item'].append(pipe.process_item)+ self.methods['process_item'].append(deferred_f_from_coro_f(pipe.process_item))def process_item(self, item, spider):return self._process_chain('process_item', item, spider)diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex bc53f5427..cfe4471d7 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -26,6 +26,13 @@ class DeferredPipeline:return d+class AsyncDefPipeline:+ async def process_item(self, item, spider):+ await defer.succeed(42)+ item['pipeline_passed'] = True+ return item++class ItemSpider(Spider):name = 'itemspider'@@ -69,3 +76,9 @@ class PipelineTestCase(unittest.TestCase):crawler = self._create_crawler(DeferredPipeline)yield crawler.crawl(mockserver=self.mockserver)self.assertEqual(len(self.items), 1)++ @defer.inlineCallbacks+ def test_asyncdef_pipeline(self):+ crawler = self._create_crawler(AsyncDefPipeline)+ yield crawler.crawl(mockserver=self.mockserver)+ self.assertEqual(len(self.items), 1)