Scrapy, a fast high-level web crawling & scraping framework for Python.
fix W291, W292, W293 (whitespaces)
Marc Hernandez Cabota5de2c64e6f1
diff --git a/pytest.ini b/pytest.iniindex f088e10ef..ac3c8cfb5 100644--- a/pytest.ini+++ b/pytest.ini@@ -85,8 +85,8 @@ flake8-ignore =scrapy/http/request/__init__.py E501scrapy/http/request/form.py E501 E123scrapy/http/request/json_request.py E501- scrapy/http/response/__init__.py E501 E128 W293 W291- scrapy/http/response/text.py E501 W293 E128 E124+ scrapy/http/response/__init__.py E501 E128+ scrapy/http/response/text.py E501 E128 E124# scrapy/linkextractorsscrapy/linkextractors/__init__.py E731 E501 E402scrapy/linkextractors/lxmlhtml.py E501 E731 E226@@ -127,9 +127,9 @@ flake8-ignore =scrapy/utils/httpobj.py E501scrapy/utils/iterators.py E501 E701scrapy/utils/log.py E128 W503- scrapy/utils/markup.py F403 W292+ scrapy/utils/markup.py F403scrapy/utils/misc.py E501 E226- scrapy/utils/multipart.py F403 W292+ scrapy/utils/multipart.py F403scrapy/utils/project.py E501scrapy/utils/python.py E501scrapy/utils/reactor.py E226@@ -144,7 +144,6 @@ flake8-ignore =scrapy/utils/url.py E501 F403 E128 F405# scrapyscrapy/__init__.py E402 E501- scrapy/_monkeypatches.py W293scrapy/cmdline.py E501scrapy/crawler.py E501scrapy/dupefilters.py E501 E202@@ -153,7 +152,7 @@ flake8-ignore =scrapy/interfaces.py E501scrapy/item.py E501 E128scrapy/link.py E501- scrapy/logformatter.py E501 W293+ scrapy/logformatter.py E501scrapy/mail.py E402 E128 E501 E502scrapy/middleware.py E128 E501scrapy/pqueues.py E501@@ -174,7 +173,7 @@ flake8-ignore =tests/test_command_parse.py E501 E128 E303 E226tests/test_command_shell.py E501 E128tests/test_commands.py E128 E501- tests/test_contracts.py E501 E128 W293+ tests/test_contracts.py E501 E128tests/test_crawl.py E501 E741 E265tests/test_crawler.py F841 E306 E501tests/test_dependencies.py F841 E501 E305@@ -189,17 +188,17 @@ flake8-ignore =tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123tests/test_downloadermiddleware_httpproxy.py E501 E128tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305- tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E303 E126+ tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126tests/test_downloadermiddleware_robotstxt.py E501tests/test_downloadermiddleware_stats.py E501- tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124+ tests/test_dupefilters.py E221 E501 E741 E128 E124tests/test_engine.py E401 E501 E128tests/test_exporters.py E501 E731 E306 E128 E124tests/test_extension_telnet.py F841tests/test_feedexport.py E501 F841 E241tests/test_http_cookies.py E501tests/test_http_headers.py E501- tests/test_http_request.py E402 E501 E127 E128 W293 E128 E126 E123+ tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123tests/test_http_response.py E501 E301 E128 E265tests/test_item.py E701 E128 F841 E306tests/test_link.py E501@@ -209,20 +208,20 @@ flake8-ignore =tests/test_mail.py E128 E501 E305tests/test_middleware.py E501 E128tests/test_pipeline_crawl.py E131 E501 E128 E126- tests/test_pipeline_files.py E501 W293 E303 E272 E226+ tests/test_pipeline_files.py E501 E303 E272 E226tests/test_pipeline_images.py F841 E501 E303tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502tests/test_proxy_connect.py E501 E741tests/test_request_cb_kwargs.py E501tests/test_responsetypes.py E501 E305- tests/test_robotstxt_interface.py E501 W291 E501+ tests/test_robotstxt_interface.py E501 E501tests/test_scheduler.py E501 E126 E123tests/test_selector.py E501 E127tests/test_spider.py E501tests/test_spidermiddleware.py E501 E226tests/test_spidermiddleware_httperror.py E128 E501 E127 E121- tests/test_spidermiddleware_offsite.py E501 E128 E111 W293- tests/test_spidermiddleware_output_chain.py E501 W293 E226+ tests/test_spidermiddleware_offsite.py E501 E128 E111+ tests/test_spidermiddleware_output_chain.py E501 E226tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121tests/test_squeues.py E501 E701 E741tests/test_utils_conf.py E501 E303 E128diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex 64e9c6c20..e79ce9acc 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -113,8 +113,8 @@ class Response(object_ref):It accepts the same arguments as ``Request.__init__`` method,but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,not only an absolute URL.-- :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`++ :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`method which supports selectors in addition to absolute/relative URLsand Link objects."""diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 1079fd6e8..4f9afde87 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -125,7 +125,7 @@ class TextResponse(Response):Return a :class:`~.Request` instance to follow a link ``url``.It accepts the same arguments as ``Request.__init__`` method,but ``url`` can be not only an absolute URL, but also-+* a relative URL;* a scrapy.link.Link object (e.g. a link extractor result);* an attribute Selector (not SelectorList) - e.g.@@ -133,7 +133,7 @@ class TextResponse(Response):``response.xpath('//img/@src')[0]``.* a Selector for ``<a>`` or ``<link>`` element, e.g.``response.css('a.my_link')[0]``.-+See :ref:`response-follow-example` for usage examples."""if isinstance(url, parsel.Selector):diff --git a/scrapy/logformatter.py b/scrapy/logformatter.pyindex 5189d7cfa..4e5963e99 100644--- a/scrapy/logformatter.py+++ b/scrapy/logformatter.py@@ -13,7 +13,7 @@ ERRORMSG = u"'Error processing %(item)s'"class LogFormatter(object):"""Class for generating log messages for different actions.-+All methods must return a dictionary listing the parameters ``level``, ``msg``and ``args`` which are going to be used for constructing the log message whencalling ``logging.log``.@@ -48,7 +48,7 @@ class LogFormatter(object):}}"""-+def crawled(self, request, response, spider):"""Logs a message when the crawler finds a webpage."""request_flags = ' %s' % str(request.flags) if request.flags else ''diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.pyindex 2455fcc16..9728c542a 100644--- a/scrapy/utils/markup.py+++ b/scrapy/utils/markup.py@@ -11,4 +11,4 @@ from w3lib.html import * # noqa: F401warnings.warn("Module `scrapy.utils.markup` is deprecated. ""Please import from `w3lib.html` instead.",- ScrapyDeprecationWarning, stacklevel=2)\ No newline at end of file+ ScrapyDeprecationWarning, stacklevel=2)diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.pyindex e81f63152..5dcf791b8 100644--- a/scrapy/utils/multipart.py+++ b/scrapy/utils/multipart.py@@ -12,4 +12,4 @@ from w3lib.form import * # noqa: F401warnings.warn("Module `scrapy.utils.multipart` is deprecated. ""If you're using `encode_multipart` function, please use ""`urllib3.filepost.encode_multipart_formdata` instead",- ScrapyDeprecationWarning, stacklevel=2)\ No newline at end of file+ ScrapyDeprecationWarning, stacklevel=2)diff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex 582e3d052..11d41c1fe 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -252,7 +252,7 @@ class ContractsManagerTest(unittest.TestCase):self.assertEqual(len(contracts), 3)self.assertEqual(frozenset(type(x) for x in contracts),frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))-+contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs)self.assertEqual(len(contracts), 3)self.assertEqual(frozenset(type(x) for x in contracts),diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex e09d66086..9c989977e 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -124,7 +124,7 @@ class MaxRetryTimesTest(unittest.TestCase):# SETTINGS: meta(max_retry_times) = 0meta_max_retry_times = 0-+req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times)@@ -137,7 +137,7 @@ class MaxRetryTimesTest(unittest.TestCase):self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)def test_with_metakey_greater(self):-+# SETINGS: RETRY_TIMES < meta(max_retry_times)self.mw.max_retry_times = 2meta_max_retry_times = 3@@ -149,7 +149,7 @@ class MaxRetryTimesTest(unittest.TestCase):self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)def test_with_metakey_lesser(self):-+# SETINGS: RETRY_TIMES > meta(max_retry_times)self.mw.max_retry_times = 5meta_max_retry_times = 4@@ -172,7 +172,7 @@ class MaxRetryTimesTest(unittest.TestCase):self._test_retry(req, DNSLookupError('foo'), 0)def _test_retry(self, req, exception, max_retry_times):-+for i in range(0, max_retry_times):req = self.mw.process_exception(req, exception, self.spider)assert isinstance(req, Request)diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex e4b0bdf83..0546558bc 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -142,12 +142,12 @@ class RFPDupeFilterTest(unittest.TestCase):r1 = Request('http://scrapytest.org/index.html')r2 = Request('http://scrapytest.org/index.html')-+dupefilter.log(r1, spider)dupefilter.log(r2, spider)assert crawler.stats.get_value('dupefilter/filtered') == 2- l.check_present(('scrapy.dupefilters', 'DEBUG',+ l.check_present(('scrapy.dupefilters', 'DEBUG',('Filtered duplicate request: <GET http://scrapytest.org/index.html>'' - no more duplicates will be shown'' (see DUPEFILTER_DEBUG to show all duplicates)')))@@ -169,7 +169,7 @@ class RFPDupeFilterTest(unittest.TestCase):r2 = Request('http://scrapytest.org/index.html',headers={'Referer': 'http://scrapytest.org/INDEX.html'})-+dupefilter.log(r1, spider)dupefilter.log(r2, spider)diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 52f2b554e..141141671 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -58,7 +58,6 @@ class FilesPipelineTestCase(unittest.TestCase):self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\//+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')-def test_fs_store(self):assert isinstance(self.pipeline.store, FSFilesStore)diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 27d79437b..24aaaf7ec 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -44,7 +44,7 @@ class BaseRobotParserTest:def test_allowed_wildcards(self):robotstxt_robotstxt_body = """User-agent: first- Disallow: /disallowed/*/end$+ Disallow: /disallowed/*/end$User-agent: secondAllow: /*alloweddiff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.pyindex 992e60be2..7511aa568 100644--- a/tests/test_spidermiddleware_offsite.py+++ b/tests/test_spidermiddleware_offsite.py@@ -73,7 +73,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3):class TestOffsiteMiddleware5(TestOffsiteMiddleware4):-+def test_get_host_regex(self):self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org']with warnings.catch_warnings(record=True) as w:diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 5b7b5e7aa..739cf1c2d 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -156,7 +156,7 @@ class GeneratorFailMiddleware:r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))yield rraise LookupError()-+def process_spider_exception(self, response, exception, spider):method = '{}.process_spider_exception'.format(self.__class__.__name__)spider.logger.info('%s: %s caught', method, exception.__class__.__name__)@@ -264,7 +264,7 @@ class TestSpiderMiddleware(TestCase):@classmethoddef tearDownClass(cls):cls.mockserver.__exit__(None, None, None)-+@defer.inlineCallbacksdef crawl_log(self, spider):crawler = get_crawler(spider)@@ -308,7 +308,7 @@ class TestSpiderMiddleware(TestCase):self.assertIn("{'from': 'errback'}", str(log1))self.assertNotIn("{'from': 'callback'}", str(log1))self.assertIn("'item_scraped_count': 1", str(log1))-+@defer.inlineCallbacksdef test_generator_callback(self):"""@@ -319,7 +319,7 @@ class TestSpiderMiddleware(TestCase):log2 = yield self.crawl_log(GeneratorCallbackSpider)self.assertIn("Middleware: ImportError exception caught", str(log2))self.assertIn("'item_scraped_count': 2", str(log2))-+@defer.inlineCallbacksdef test_not_a_generator_callback(self):"""