Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge back tests/py36/_test_crawl.py.
Andrey Rakhmatullinde640f41ecfa
diff --git a/conftest.py b/conftest.pyindex b39d644a5..be97b7714 100644--- a/conftest.py+++ b/conftest.py@@ -14,8 +14,6 @@ collect_ignore = [*_py_files("tests/CrawlerProcess"),# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess*_py_files("tests/CrawlerRunner"),- # Py36-only parts of respective tests- *_py_files("tests/py36"),]for line in open('tests/ignores.txt'):diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.pydeleted file mode 100644index 162a53760..000000000--- a/tests/py36/_test_crawl.py+++ /dev/null@@ -1,57 +0,0 @@-import asyncio--from scrapy import Request-from tests.spiders import SimpleSpider---class AsyncDefAsyncioGenSpider(SimpleSpider):-- name = 'asyncdef_asyncio_gen'-- async def parse(self, response):- await asyncio.sleep(0.2)- yield {'foo': 42}- self.logger.info("Got response %d" % response.status)---class AsyncDefAsyncioGenLoopSpider(SimpleSpider):-- name = 'asyncdef_asyncio_gen_loop'-- async def parse(self, response):- for i in range(10):- await asyncio.sleep(0.1)- yield {'foo': i}- self.logger.info("Got response %d" % response.status)---class AsyncDefAsyncioGenComplexSpider(SimpleSpider):-- name = 'asyncdef_asyncio_gen_complex'- initial_reqs = 4- following_reqs = 3- depth = 2-- def _get_req(self, index, cb=None):- return Request(self.mockserver.url("/status?n=200&request=%d" % index),- meta={'index': index},- dont_filter=True,- callback=cb)-- def start_requests(self):- for i in range(1, self.initial_reqs + 1):- yield self._get_req(i)-- async def parse(self, response):- index = response.meta['index']- yield {'index': index}- if index < 10 ** self.depth:- for new_index in range(10 * index, 10 * index + self.following_reqs):- yield self._get_req(new_index)- yield self._get_req(index, cb=self.parse2)- await asyncio.sleep(0.1)- yield {'index': index + 5}-- async def parse2(self, response):- await asyncio.sleep(0.1)- yield {'index2': response.meta['index']}diff --git a/tests/spiders.py b/tests/spiders.pyindex 8a0ee44b7..8a85d2b51 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -148,6 +148,59 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):return reqs+class AsyncDefAsyncioGenSpider(SimpleSpider):++ name = 'asyncdef_asyncio_gen'++ async def parse(self, response):+ await asyncio.sleep(0.2)+ yield {'foo': 42}+ self.logger.info("Got response %d" % response.status)+++class AsyncDefAsyncioGenLoopSpider(SimpleSpider):++ name = 'asyncdef_asyncio_gen_loop'++ async def parse(self, response):+ for i in range(10):+ await asyncio.sleep(0.1)+ yield {'foo': i}+ self.logger.info("Got response %d" % response.status)+++class AsyncDefAsyncioGenComplexSpider(SimpleSpider):++ name = 'asyncdef_asyncio_gen_complex'+ initial_reqs = 4+ following_reqs = 3+ depth = 2++ def _get_req(self, index, cb=None):+ return Request(self.mockserver.url("/status?n=200&request=%d" % index),+ meta={'index': index},+ dont_filter=True,+ callback=cb)++ def start_requests(self):+ for i in range(1, self.initial_reqs + 1):+ yield self._get_req(i)++ async def parse(self, response):+ index = response.meta['index']+ yield {'index': index}+ if index < 10 ** self.depth:+ for new_index in range(10 * index, 10 * index + self.following_reqs):+ yield self._get_req(new_index)+ yield self._get_req(index, cb=self.parse2)+ await asyncio.sleep(0.1)+ yield {'index': index + 5}++ async def parse2(self, response):+ await asyncio.sleep(0.1)+ yield {'index2': response.meta['index']}++class ItemSpider(FollowAllSpider):name = 'item'diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex c1b918baf..ba8c3fd3c 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -19,6 +19,9 @@ from scrapy.http.response import Responsefrom scrapy.utils.python import to_unicodefrom tests.mockserver import MockServerfrom tests.spiders import (+ AsyncDefAsyncioGenComplexSpider,+ AsyncDefAsyncioGenLoopSpider,+ AsyncDefAsyncioGenSpider,AsyncDefAsyncioReqsReturnSpider,AsyncDefAsyncioReturnSingleElementSpider,AsyncDefAsyncioReturnSpider,@@ -407,7 +410,6 @@ class CrawlSpiderTestCase(TestCase):@mark.only_asyncio()@defer.inlineCallbacksdef test_async_def_asyncgen_parse(self):- from tests.py36._test_crawl import AsyncDefAsyncioGenSpidercrawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider)with LogCapture() as log:yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)@@ -423,7 +425,6 @@ class CrawlSpiderTestCase(TestCase):def _on_item_scraped(item):items.append(item)- from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpidercrawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider)crawler.signals.connect(_on_item_scraped, signals.item_scraped)with LogCapture() as log:@@ -442,7 +443,6 @@ class CrawlSpiderTestCase(TestCase):def _on_item_scraped(item):items.append(item)- from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpidercrawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider)crawler.signals.connect(_on_item_scraped, signals.item_scraped)yield crawler.crawl(mockserver=self.mockserver)