Scrapy, a fast high-level web crawling & scraping framework for Python.
Also test non-default async callbacks.
Andrey Rakhmatullin36dcf9018490
diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.pyindex 74c7daf53..162a53760 100644--- a/tests/py36/_test_crawl.py+++ b/tests/py36/_test_crawl.py@@ -32,12 +32,14 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):following_reqs = 3depth = 2- def _get_req(self, index):+ def _get_req(self, index, cb=None):return Request(self.mockserver.url("/status?n=200&request=%d" % index),- meta={'index': index})+ meta={'index': index},+ dont_filter=True,+ callback=cb)def start_requests(self):- for i in range(self.initial_reqs):+ for i in range(1, self.initial_reqs + 1):yield self._get_req(i)async def parse(self, response):@@ -46,5 +48,10 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):if index < 10 ** self.depth:for new_index in range(10 * index, 10 * index + self.following_reqs):yield self._get_req(new_index)+ yield self._get_req(index, cb=self.parse2)await asyncio.sleep(0.1)yield {'index': index + 5}++ async def parse2(self, response):+ await asyncio.sleep(0.1)+ yield {'index2': response.meta['index']}diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 626000147..64819acb6 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -392,9 +392,12 @@ with multiples linescrawler.signals.connect(_on_item_scraped, signals.item_scraped)yield crawler.crawl(mockserver=self.mockserver)itemcount = crawler.stats.get_value('item_scraped_count')- self.assertEqual(itemcount, 80)- for i in [0, 3, 21, 22, 207, 311]: # some random items+ self.assertEqual(itemcount, 156)+ # some random items+ for i in [1, 4, 21, 22, 207, 311]:self.assertIn({'index': i}, items)+ for i in [10, 30, 122]:+ self.assertIn({'index2': i}, items)@mark.only_asyncio()@defer.inlineCallbacks