Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge back tests/py36/_test_crawl.py.

Andrey Rakhmatullinde640f41ecfa
diff --git a/conftest.py b/conftest.py
index b39d644a5..be97b7714 100644
--- a/conftest.py
+++ b/conftest.py
@@ -14,8 +14,6 @@ collect_ignore = [
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
- # Py36-only parts of respective tests
- *_py_files("tests/py36"),
]
for line in open('tests/ignores.txt'):
diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py
deleted file mode 100644
index 162a53760..000000000
--- a/tests/py36/_test_crawl.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import asyncio
-
-from scrapy import Request
-from tests.spiders import SimpleSpider
-
-
-class AsyncDefAsyncioGenSpider(SimpleSpider):
-
- name = 'asyncdef_asyncio_gen'
-
- async def parse(self, response):
- await asyncio.sleep(0.2)
- yield {'foo': 42}
- self.logger.info("Got response %d" % response.status)
-
-
-class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
-
- name = 'asyncdef_asyncio_gen_loop'
-
- async def parse(self, response):
- for i in range(10):
- await asyncio.sleep(0.1)
- yield {'foo': i}
- self.logger.info("Got response %d" % response.status)
-
-
-class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
-
- name = 'asyncdef_asyncio_gen_complex'
- initial_reqs = 4
- following_reqs = 3
- depth = 2
-
- def _get_req(self, index, cb=None):
- return Request(self.mockserver.url("/status?n=200&request=%d" % index),
- meta={'index': index},
- dont_filter=True,
- callback=cb)
-
- def start_requests(self):
- for i in range(1, self.initial_reqs + 1):
- yield self._get_req(i)
-
- async def parse(self, response):
- index = response.meta['index']
- yield {'index': index}
- if index < 10 ** self.depth:
- for new_index in range(10 * index, 10 * index + self.following_reqs):
- yield self._get_req(new_index)
- yield self._get_req(index, cb=self.parse2)
- await asyncio.sleep(0.1)
- yield {'index': index + 5}
-
- async def parse2(self, response):
- await asyncio.sleep(0.1)
- yield {'index2': response.meta['index']}
diff --git a/tests/spiders.py b/tests/spiders.py
index 8a0ee44b7..8a85d2b51 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -148,6 +148,59 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
return reqs
+class AsyncDefAsyncioGenSpider(SimpleSpider):
+
+ name = 'asyncdef_asyncio_gen'
+
+ async def parse(self, response):
+ await asyncio.sleep(0.2)
+ yield {'foo': 42}
+ self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
+
+ name = 'asyncdef_asyncio_gen_loop'
+
+ async def parse(self, response):
+ for i in range(10):
+ await asyncio.sleep(0.1)
+ yield {'foo': i}
+ self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
+
+ name = 'asyncdef_asyncio_gen_complex'
+ initial_reqs = 4
+ following_reqs = 3
+ depth = 2
+
+ def _get_req(self, index, cb=None):
+ return Request(self.mockserver.url("/status?n=200&request=%d" % index),
+ meta={'index': index},
+ dont_filter=True,
+ callback=cb)
+
+ def start_requests(self):
+ for i in range(1, self.initial_reqs + 1):
+ yield self._get_req(i)
+
+ async def parse(self, response):
+ index = response.meta['index']
+ yield {'index': index}
+ if index < 10 ** self.depth:
+ for new_index in range(10 * index, 10 * index + self.following_reqs):
+ yield self._get_req(new_index)
+ yield self._get_req(index, cb=self.parse2)
+ await asyncio.sleep(0.1)
+ yield {'index': index + 5}
+
+ async def parse2(self, response):
+ await asyncio.sleep(0.1)
+ yield {'index2': response.meta['index']}
+
+
class ItemSpider(FollowAllSpider):
name = 'item'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index c1b918baf..ba8c3fd3c 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -19,6 +19,9 @@ from scrapy.http.response import Response
from scrapy.utils.python import to_unicode
from tests.mockserver import MockServer
from tests.spiders import (
+ AsyncDefAsyncioGenComplexSpider,
+ AsyncDefAsyncioGenLoopSpider,
+ AsyncDefAsyncioGenSpider,
AsyncDefAsyncioReqsReturnSpider,
AsyncDefAsyncioReturnSingleElementSpider,
AsyncDefAsyncioReturnSpider,
@@ -407,7 +410,6 @@ class CrawlSpiderTestCase(TestCase):
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncgen_parse(self):
- from tests.py36._test_crawl import AsyncDefAsyncioGenSpider
crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@@ -423,7 +425,6 @@ class CrawlSpiderTestCase(TestCase):
def _on_item_scraped(item):
items.append(item)
- from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider
crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
@@ -442,7 +443,6 @@ class CrawlSpiderTestCase(TestCase):
def _on_item_scraped(item):
items.append(item)
- from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider
crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
yield crawler.crawl(mockserver=self.mockserver)