Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Reducing amount of warnings during test run (#5162)

Vostretsov Nikita23cfdb058e80
diff --git a/.flake8 b/.flake8
new file mode 100644
index 000000000..1c503fb0b
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,19 @@
+[flake8]
+
+max-line-length = 119
+ignore = W503
+
+exclude =
+# Exclude files that are meant to provide top-level imports
+# E402: Module level import not at top of file
+# F401: Module imported but unused
+ scrapy/__init__.py E402
+ scrapy/core/downloader/handlers/http.py F401
+ scrapy/http/__init__.py F401
+ scrapy/linkextractors/__init__.py E402 F401
+ scrapy/selector/__init__.py F401
+ scrapy/spiders/__init__.py E402 F401
+
+ # Issues pending a review:
+ scrapy/utils/url.py F403 F405
+ tests/test_loader.py E741
diff --git a/.gitignore b/.gitignore
index 795e2605e..d77d24624 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,8 @@ htmlcov/
.coverage
.pytest_cache/
.coverage.*
+coverage.*
+test-output.*
.cache/
.mypy_cache/
/tests/keys/localhost.crt
diff --git a/conftest.py b/conftest.py
index 4931c5a79..05b4ccdad 100644
--- a/conftest.py
+++ b/conftest.py
@@ -21,10 +21,11 @@ collect_ignore = [
*_py_files("tests/CrawlerRunner"),
]
-for line in open('tests/ignores.txt'):
- file_path = line.strip()
- if file_path and file_path[0] != '#':
- collect_ignore.append(file_path)
+with open('tests/ignores.txt') as reader:
+ for line in reader:
+ file_path = line.strip()
+ if file_path and file_path[0] != '#':
+ collect_ignore.append(file_path)
if not H2_ENABLED:
collect_ignore.extend(
diff --git a/pytest.ini b/pytest.ini
index 0aae09ff5..6de08c78d 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -20,20 +20,5 @@ addopts =
--ignore=docs/utils
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
-flake8-max-line-length = 119
-flake8-ignore =
- W503
-
- # Exclude files that are meant to provide top-level imports
- # E402: Module level import not at top of file
- # F401: Module imported but unused
- scrapy/__init__.py E402
- scrapy/core/downloader/handlers/http.py F401
- scrapy/http/__init__.py F401
- scrapy/linkextractors/__init__.py E402 F401
- scrapy/selector/__init__.py F401
- scrapy/spiders/__init__.py E402 F401
-
- # Issues pending a review:
- scrapy/utils/url.py F403 F405
- tests/test_loader.py E741
+filterwarnings=
+ ignore::DeprecationWarning:twisted.web.test.test_webclient
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index ebc477e74..04bae31d3 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -6,12 +6,14 @@ import tempfile
import unittest
from io import BytesIO
from datetime import datetime
+from warnings import catch_warnings, filterwarnings
import lxml.etree
from itemadapter import ItemAdapter
from scrapy.item import Item, Field
from scrapy.utils.python import to_unicode
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.exporters import (
BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter,
XmlItemExporter, JsonLinesItemExporter, JsonItemExporter,
@@ -172,10 +174,12 @@ class PythonItemExporterTest(BaseItemExporterTest):
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
def test_export_binary(self):
- exporter = PythonItemExporter(binary=True)
- value = self.item_class(name='John\xa3', age='22')
- expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
- self.assertEqual(expected, exporter.export_item(value))
+ with catch_warnings():
+ filterwarnings('ignore', category=ScrapyDeprecationWarning)
+ exporter = PythonItemExporter(binary=True)
+ value = self.item_class(name='John\xa3', age='22')
+ expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
+ self.assertEqual(expected, exporter.export_item(value))
def test_nonstring_types_item(self):
item = self._get_nonstring_types_item()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d248824fc..df7ec4461 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -515,7 +515,7 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
class DummyBlockingFeedStorage(BlockingFeedStorage):
- def __init__(self, uri):
+ def __init__(self, uri, *args, feed_options=None):
self.path = file_uri_to_path(uri)
def _store_in_thread(self, file):
@@ -541,7 +541,7 @@ class LogOnStoreFileStorage:
It can be used to make sure `store` method is invoked.
"""
- def __init__(self, uri):
+ def __init__(self, uri, feed_options=None):
self.path = file_uri_to_path(uri)
self.logger = getLogger()
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index f831ef5dc..04a594d03 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,6 +1,6 @@
import unittest
from unittest import mock
-from warnings import catch_warnings
+from warnings import catch_warnings, filterwarnings
from w3lib.encoding import resolve_encoding
@@ -134,7 +134,9 @@ class BaseResponseTest(unittest.TestCase):
assert isinstance(response.text, str)
self._assert_response_encoding(response, encoding)
self.assertEqual(response.body, body_bytes)
- self.assertEqual(response.body_as_unicode(), body_unicode)
+ with catch_warnings():
+ filterwarnings("ignore", category=ScrapyDeprecationWarning)
+ self.assertEqual(response.body_as_unicode(), body_unicode)
self.assertEqual(response.text, body_unicode)
def _assert_response_encoding(self, response, encoding):
@@ -345,8 +347,10 @@ class TextResponseTest(BaseResponseTest):
r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
# check body_as_unicode
- self.assertTrue(isinstance(r1.body_as_unicode(), str))
- self.assertEqual(r1.body_as_unicode(), unicode_string)
+ with catch_warnings():
+ filterwarnings("ignore", category=ScrapyDeprecationWarning)
+ self.assertTrue(isinstance(r1.body_as_unicode(), str))
+ self.assertEqual(r1.body_as_unicode(), unicode_string)
# check response.text
self.assertTrue(isinstance(r1.text, str))
diff --git a/tests/test_item.py b/tests/test_item.py
index 78d204e34..c94bb44af 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,6 +1,6 @@
import unittest
from unittest import mock
-from warnings import catch_warnings
+from warnings import catch_warnings, filterwarnings
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta
@@ -328,16 +328,18 @@ class BaseItemTest(unittest.TestCase):
class SubclassedItem(Item):
pass
- self.assertTrue(isinstance(BaseItem(), BaseItem))
- self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
- self.assertTrue(isinstance(Item(), BaseItem))
- self.assertTrue(isinstance(SubclassedItem(), BaseItem))
-
- # make sure internal checks using private _BaseItem class succeed
- self.assertTrue(isinstance(BaseItem(), _BaseItem))
- self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
- self.assertTrue(isinstance(Item(), _BaseItem))
- self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
+ with catch_warnings():
+ filterwarnings("ignore", category=ScrapyDeprecationWarning)
+ self.assertTrue(isinstance(BaseItem(), BaseItem))
+ self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))
+ self.assertTrue(isinstance(Item(), BaseItem))
+ self.assertTrue(isinstance(SubclassedItem(), BaseItem))
+
+ # make sure internal checks using private _BaseItem class succeed
+ self.assertTrue(isinstance(BaseItem(), _BaseItem))
+ self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))
+ self.assertTrue(isinstance(Item(), _BaseItem))
+ self.assertTrue(isinstance(SubclassedItem(), _BaseItem))
def test_deprecation_warning(self):
"""
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 35d35b45d..e47afa266 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -108,7 +108,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
# ignore subclassing warnings
with warnings.catch_warnings():
- warnings.simplefilter('ignore', ScrapyDeprecationWarning)
+ warnings.simplefilter('ignore', MyWarning)
class UserClass(Deprecated):
pass
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3115cc92f..4b3964154 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -5,8 +5,9 @@ import platform
import unittest
from datetime import datetime
from itertools import count
-from warnings import catch_warnings
+from warnings import catch_warnings, filterwarnings
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.python import (
memoizemethod_noargs, binary_is_text, equal_attributes,
WeakKeyCache, get_func_args, to_bytes, to_unicode,
@@ -160,7 +161,11 @@ class UtilsPythonTestCase(unittest.TestCase):
pass
_values = count()
- wk = WeakKeyCache(lambda k: next(_values))
+
+ with catch_warnings():
+ filterwarnings("ignore", category=ScrapyDeprecationWarning)
+ wk = WeakKeyCache(lambda k: next(_values))
+
k = _Weakme()
v = wk[k]
self.assertEqual(v, wk[k])