Scrapy, a fast high-level web crawling & scraping framework for Python.
Reducing amount of warnings during test run (#5162)
Vostretsov Nikita23cfdb058e80
diff --git a/.flake8 b/.flake8new file mode 100644index 000000000..1c503fb0b--- /dev/null+++ b/.flake8@@ -0,0 +1,19 @@+[flake8]++max-line-length = 119+ignore = W503++exclude =+# Exclude files that are meant to provide top-level imports+# E402: Module level import not at top of file+# F401: Module imported but unused+ scrapy/__init__.py E402+ scrapy/core/downloader/handlers/http.py F401+ scrapy/http/__init__.py F401+ scrapy/linkextractors/__init__.py E402 F401+ scrapy/selector/__init__.py F401+ scrapy/spiders/__init__.py E402 F401++ # Issues pending a review:+ scrapy/utils/url.py F403 F405+ tests/test_loader.py E741diff --git a/.gitignore b/.gitignoreindex 795e2605e..d77d24624 100644--- a/.gitignore+++ b/.gitignore@@ -14,6 +14,8 @@ htmlcov/.coverage.pytest_cache/.coverage.*+coverage.*+test-output.*.cache/.mypy_cache//tests/keys/localhost.crtdiff --git a/conftest.py b/conftest.pyindex 4931c5a79..05b4ccdad 100644--- a/conftest.py+++ b/conftest.py@@ -21,10 +21,11 @@ collect_ignore = [*_py_files("tests/CrawlerRunner"),]-for line in open('tests/ignores.txt'):- file_path = line.strip()- if file_path and file_path[0] != '#':- collect_ignore.append(file_path)+with open('tests/ignores.txt') as reader:+ for line in reader:+ file_path = line.strip()+ if file_path and file_path[0] != '#':+ collect_ignore.append(file_path)if not H2_ENABLED:collect_ignore.extend(diff --git a/pytest.ini b/pytest.iniindex 0aae09ff5..6de08c78d 100644--- a/pytest.ini+++ b/pytest.ini@@ -20,20 +20,5 @@ addopts =--ignore=docs/utilsmarkers =only_asyncio: marks tests as only enabled when --reactor=asyncio is passed-flake8-max-line-length = 119-flake8-ignore =- W503-- # Exclude files that are meant to provide top-level imports- # E402: Module level import not at top of file- # F401: Module imported but unused- scrapy/__init__.py E402- scrapy/core/downloader/handlers/http.py F401- scrapy/http/__init__.py F401- scrapy/linkextractors/__init__.py E402 F401- scrapy/selector/__init__.py F401- scrapy/spiders/__init__.py E402 F401-- # Issues pending a review:- scrapy/utils/url.py F403 F405- tests/test_loader.py E741+filterwarnings=+ ignore::DeprecationWarning:twisted.web.test.test_webclientdiff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex ebc477e74..04bae31d3 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -6,12 +6,14 @@ import tempfileimport unittestfrom io import BytesIOfrom datetime import datetime+from warnings import catch_warnings, filterwarningsimport lxml.etreefrom itemadapter import ItemAdapterfrom scrapy.item import Item, Fieldfrom scrapy.utils.python import to_unicode+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.exporters import (BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter,XmlItemExporter, JsonLinesItemExporter, JsonItemExporter,@@ -172,10 +174,12 @@ class PythonItemExporterTest(BaseItemExporterTest):self.assertEqual(type(exported['age'][0]['age'][0]), dict)def test_export_binary(self):- exporter = PythonItemExporter(binary=True)- value = self.item_class(name='John\xa3', age='22')- expected = {b'name': b'John\xc2\xa3', b'age': b'22'}- self.assertEqual(expected, exporter.export_item(value))+ with catch_warnings():+ filterwarnings('ignore', category=ScrapyDeprecationWarning)+ exporter = PythonItemExporter(binary=True)+ value = self.item_class(name='John\xa3', age='22')+ expected = {b'name': b'John\xc2\xa3', b'age': b'22'}+ self.assertEqual(expected, exporter.export_item(value))def test_nonstring_types_item(self):item = self._get_nonstring_types_item()diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex d248824fc..df7ec4461 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -515,7 +515,7 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):class DummyBlockingFeedStorage(BlockingFeedStorage):- def __init__(self, uri):+ def __init__(self, uri, *args, feed_options=None):self.path = file_uri_to_path(uri)def _store_in_thread(self, file):@@ -541,7 +541,7 @@ class LogOnStoreFileStorage:It can be used to make sure `store` method is invoked."""- def __init__(self, uri):+ def __init__(self, uri, feed_options=None):self.path = file_uri_to_path(uri)self.logger = getLogger()diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex f831ef5dc..04a594d03 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -1,6 +1,6 @@import unittestfrom unittest import mock-from warnings import catch_warnings+from warnings import catch_warnings, filterwarningsfrom w3lib.encoding import resolve_encoding@@ -134,7 +134,9 @@ class BaseResponseTest(unittest.TestCase):assert isinstance(response.text, str)self._assert_response_encoding(response, encoding)self.assertEqual(response.body, body_bytes)- self.assertEqual(response.body_as_unicode(), body_unicode)+ with catch_warnings():+ filterwarnings("ignore", category=ScrapyDeprecationWarning)+ self.assertEqual(response.body_as_unicode(), body_unicode)self.assertEqual(response.text, body_unicode)def _assert_response_encoding(self, response, encoding):@@ -345,8 +347,10 @@ class TextResponseTest(BaseResponseTest):r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')# check body_as_unicode- self.assertTrue(isinstance(r1.body_as_unicode(), str))- self.assertEqual(r1.body_as_unicode(), unicode_string)+ with catch_warnings():+ filterwarnings("ignore", category=ScrapyDeprecationWarning)+ self.assertTrue(isinstance(r1.body_as_unicode(), str))+ self.assertEqual(r1.body_as_unicode(), unicode_string)# check response.textself.assertTrue(isinstance(r1.text, str))diff --git a/tests/test_item.py b/tests/test_item.pyindex 78d204e34..c94bb44af 100644--- a/tests/test_item.py+++ b/tests/test_item.py@@ -1,6 +1,6 @@import unittestfrom unittest import mock-from warnings import catch_warnings+from warnings import catch_warnings, filterwarningsfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta@@ -328,16 +328,18 @@ class BaseItemTest(unittest.TestCase):class SubclassedItem(Item):pass- self.assertTrue(isinstance(BaseItem(), BaseItem))- self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))- self.assertTrue(isinstance(Item(), BaseItem))- self.assertTrue(isinstance(SubclassedItem(), BaseItem))-- # make sure internal checks using private _BaseItem class succeed- self.assertTrue(isinstance(BaseItem(), _BaseItem))- self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))- self.assertTrue(isinstance(Item(), _BaseItem))- self.assertTrue(isinstance(SubclassedItem(), _BaseItem))+ with catch_warnings():+ filterwarnings("ignore", category=ScrapyDeprecationWarning)+ self.assertTrue(isinstance(BaseItem(), BaseItem))+ self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem))+ self.assertTrue(isinstance(Item(), BaseItem))+ self.assertTrue(isinstance(SubclassedItem(), BaseItem))++ # make sure internal checks using private _BaseItem class succeed+ self.assertTrue(isinstance(BaseItem(), _BaseItem))+ self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem))+ self.assertTrue(isinstance(Item(), _BaseItem))+ self.assertTrue(isinstance(SubclassedItem(), _BaseItem))def test_deprecation_warning(self):"""diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.pyindex 35d35b45d..e47afa266 100644--- a/tests/test_utils_deprecate.py+++ b/tests/test_utils_deprecate.py@@ -108,7 +108,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):# ignore subclassing warningswith warnings.catch_warnings():- warnings.simplefilter('ignore', ScrapyDeprecationWarning)+ warnings.simplefilter('ignore', MyWarning)class UserClass(Deprecated):passdiff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex 3115cc92f..4b3964154 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -5,8 +5,9 @@ import platformimport unittestfrom datetime import datetimefrom itertools import count-from warnings import catch_warnings+from warnings import catch_warnings, filterwarnings+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.python import (memoizemethod_noargs, binary_is_text, equal_attributes,WeakKeyCache, get_func_args, to_bytes, to_unicode,@@ -160,7 +161,11 @@ class UtilsPythonTestCase(unittest.TestCase):pass_values = count()- wk = WeakKeyCache(lambda k: next(_values))++ with catch_warnings():+ filterwarnings("ignore", category=ScrapyDeprecationWarning)+ wk = WeakKeyCache(lambda k: next(_values))+k = _Weakme()v = wk[k]self.assertEqual(v, wk[k])