Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Also test non-default async callbacks.

Andrey Rakhmatullin36dcf9018490
diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py
index 74c7daf53..162a53760 100644
--- a/tests/py36/_test_crawl.py
+++ b/tests/py36/_test_crawl.py
@@ -32,12 +32,14 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
following_reqs = 3
depth = 2
- def _get_req(self, index):
+ def _get_req(self, index, cb=None):
return Request(self.mockserver.url("/status?n=200&request=%d" % index),
- meta={'index': index})
+ meta={'index': index},
+ dont_filter=True,
+ callback=cb)
def start_requests(self):
- for i in range(self.initial_reqs):
+ for i in range(1, self.initial_reqs + 1):
yield self._get_req(i)
async def parse(self, response):
@@ -46,5 +48,10 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
if index < 10 ** self.depth:
for new_index in range(10 * index, 10 * index + self.following_reqs):
yield self._get_req(new_index)
+ yield self._get_req(index, cb=self.parse2)
await asyncio.sleep(0.1)
yield {'index': index + 5}
+
+ async def parse2(self, response):
+ await asyncio.sleep(0.1)
+ yield {'index2': response.meta['index']}
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 626000147..64819acb6 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -392,9 +392,12 @@ with multiples lines
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
yield crawler.crawl(mockserver=self.mockserver)
itemcount = crawler.stats.get_value('item_scraped_count')
- self.assertEqual(itemcount, 80)
- for i in [0, 3, 21, 22, 207, 311]: # some random items
+ self.assertEqual(itemcount, 156)
+ # some random items
+ for i in [1, 4, 21, 22, 207, 311]:
self.assertIn({'index': i}, items)
+ for i in [10, 30, 122]:
+ self.assertIn({'index2': i}, items)
@mark.only_asyncio()
@defer.inlineCallbacks