Scrapy, a fast high-level web crawling & scraping framework for Python.
Keep 2 spaces between code and inline comments (#4195)
Adrián Chavesb5c4c2cae897
diff --git a/pytest.ini b/pytest.iniindex 33c34b8e8..1b23595c0 100644--- a/pytest.ini+++ b/pytest.ini@@ -26,8 +26,8 @@ flake8-ignore =scrapy/http/__init__.py F401# Issues pending a review:# extras- extras/qps-bench-server.py E261 E501- extras/qpsclient.py E501 E261 E501+ extras/qps-bench-server.py E501+ extras/qpsclient.py E501 E501# scrapy/commandsscrapy/commands/__init__.py E128 E501scrapy/commands/check.py E501@@ -45,15 +45,15 @@ flake8-ignore =scrapy/contracts/__init__.py E501 W504scrapy/contracts/default.py E502 E128# scrapy/core- scrapy/core/engine.py E261 E501 E128 E127 E306 E502+ scrapy/core/engine.py E501 E128 E127 E306 E502scrapy/core/scheduler.py E501- scrapy/core/scraper.py E501 E306 E261 E128 W504+ scrapy/core/scraper.py E501 E306 E128 W504scrapy/core/spidermw.py E501 E731 E502 E126 E226scrapy/core/downloader/__init__.py E501scrapy/core/downloader/contextfactory.py E501 E128 E126scrapy/core/downloader/middleware.py E501 E502scrapy/core/downloader/tls.py E501 E305 E241- scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226+ scrapy/core/downloader/webclient.py E731 E501 E502 E128 E126 E226scrapy/core/downloader/handlers/__init__.py E501scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127scrapy/core/downloader/handlers/http10.py E501@@ -102,7 +102,7 @@ flake8-ignore =scrapy/selector/unified.py E501 E111# scrapy/settingsscrapy/settings/__init__.py E501- scrapy/settings/default_settings.py E501 E261 E114 E116 E226+ scrapy/settings/default_settings.py E501 E114 E116 E226scrapy/settings/deprecated.py E501# scrapy/spidermiddlewaresscrapy/spidermiddlewares/httperror.py E501@@ -112,17 +112,16 @@ flake8-ignore =# scrapy/spidersscrapy/spiders/__init__.py E501 E402scrapy/spiders/crawl.py E501- scrapy/spiders/feed.py E501 E261+ scrapy/spiders/feed.py E501scrapy/spiders/sitemap.py E501# scrapy/utilsscrapy/utils/benchserver.py E501scrapy/utils/conf.py E402 E502 E501- scrapy/utils/console.py E261 E306 E305+ scrapy/utils/console.py E306 E305scrapy/utils/datatypes.py E501 E226scrapy/utils/decorators.py E501scrapy/utils/defer.py E501 E128scrapy/utils/deprecate.py E128 E501 E127 E502- scrapy/utils/engine.py E261scrapy/utils/gz.py E305 E501 W504scrapy/utils/http.py F403 E226scrapy/utils/httpobj.py E501@@ -150,7 +149,7 @@ flake8-ignore =scrapy/crawler.py E501scrapy/dupefilters.py E501 E202scrapy/exceptions.py E501- scrapy/exporters.py E501 E261 E226+ scrapy/exporters.py E501 E226scrapy/interfaces.py E501scrapy/item.py E501 E128scrapy/link.py E501@@ -171,7 +170,7 @@ flake8-ignore =tests/pipelines.py F841 E226tests/spiders.py E501 E127tests/test_closespider.py E501 E127- tests/test_command_fetch.py E501 E261+ tests/test_command_fetch.py E501tests/test_command_parse.py E501 E128 E303 E226tests/test_command_shell.py E501 E128tests/test_commands.py E128 E501@@ -179,7 +178,7 @@ flake8-ignore =tests/test_crawl.py E501 E741 E265tests/test_crawler.py F841 E306 E501tests/test_dependencies.py F841 E501 E305- tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123+ tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E502 E701 E126 E226 E123tests/test_downloadermiddleware.py E501tests/test_downloadermiddleware_ajaxcrawlable.py E501tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126@@ -194,13 +193,13 @@ flake8-ignore =tests/test_downloadermiddleware_robotstxt.py E501tests/test_downloadermiddleware_stats.py E501tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124- tests/test_engine.py E401 E501 E502 E128 E261+ tests/test_engine.py E401 E501 E502 E128tests/test_exporters.py E501 E731 E306 E128 E124tests/test_extension_telnet.py F841tests/test_feedexport.py E501 F841 E241tests/test_http_cookies.py E501tests/test_http_headers.py E501- tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123+ tests/test_http_request.py E402 E501 E127 E128 W293 E502 E128 E502 E126 E123tests/test_http_response.py E501 E301 E502 E128 E265tests/test_item.py E701 E128 F841 E306tests/test_link.py E501@@ -212,7 +211,7 @@ flake8-ignore =tests/test_pipeline_crawl.py E131 E501 E128 E126tests/test_pipeline_files.py E501 W293 E303 E272 E226tests/test_pipeline_images.py F841 E501 E303- tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502+ tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502tests/test_proxy_connect.py E501 E741tests/test_request_cb_kwargs.py E501tests/test_responsetypes.py E501 E305@@ -224,12 +223,12 @@ flake8-ignore =tests/test_spidermiddleware_httperror.py E128 E501 E127 E121tests/test_spidermiddleware_offsite.py E501 E128 E111 W293tests/test_spidermiddleware_output_chain.py E501 W293 E226- tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121+ tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121tests/test_squeues.py E501 E701 E741tests/test_utils_conf.py E501 E303 E128tests/test_utils_curl.py E501tests/test_utils_datatypes.py E402 E501 E305- tests/test_utils_defer.py E306 E261 E501 F841 E226+ tests/test_utils_defer.py E306 E501 F841 E226tests/test_utils_deprecate.py F841 E306 E501tests/test_utils_http.py E501 E502 E128 W504tests/test_utils_iterators.py E501 E128 E129 E303 E241@@ -240,7 +239,7 @@ flake8-ignore =tests/test_utils_response.py E501tests/test_utils_signal.py E741 F841 E731 E226tests/test_utils_sitemap.py E128 E501 E124- tests/test_utils_spider.py E261 E305+ tests/test_utils_spider.py E305tests/test_utils_template.py E305tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.pyindex 798346f19..f368c3bae 100644--- a/scrapy/core/downloader/webclient.py+++ b/scrapy/core/downloader/webclient.py@@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient):delimiter = b'\n'def connectionMade(self):- self.headers = Headers() # bucket for response headers+ self.headers = Headers() # bucket for response headers# Method commandself.sendCommand(self.factory.method, self.factory.path)diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex fa913e528..829e69993 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -25,7 +25,7 @@ class Slot(object):def __init__(self, start_requests, close_if_idle, nextcall, scheduler):self.closing = False- self.inprogress = set() # requests in progress+ self.inprogress = set() # requests in progressself.start_requests = iter(start_requests)self.close_if_idle = close_if_idleself.nextcall = nextcalldiff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex db463f989..b3d585cce 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -123,7 +123,7 @@ class Scraper(object):callback/errback"""assert isinstance(response, (Response, Failure))- dfd = self._scrape2(response, request, spider) # returns spiders processed output+ dfd = self._scrape2(response, request, spider) # returns spiders processed outputdfd.addErrback(self.handle_spider_error, request, response, spider)dfd.addCallback(self.handle_spider_output, request, response, spider)return dfddiff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex fcb55da67..5bf131312 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -199,7 +199,7 @@ class CsvItemExporter(BaseItemExporter):line_buffering=False,write_through=True,encoding=self.encoding,- newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034+ newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034)self.csv_writer = csv.writer(self.stream, **kwargs)self._headers_not_written = Truediff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 5c9678c01..1e163e1fb 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -84,8 +84,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT'-DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,- # also allowing negotiation+# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation:+DOWNLOADER_CLIENT_TLS_METHOD = 'TLS'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = FalseDOWNLOADER_MIDDLEWARES = {}diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.pyindex 197812a26..c566f0236 100644--- a/scrapy/spiders/feed.py+++ b/scrapy/spiders/feed.py@@ -100,8 +100,8 @@ class CSVFeedSpider(Spider):and the file's headers."""- delimiter = None # When this is None, python's csv module's default delimiter is used- quotechar = None # When this is None, python's csv module's default quotechar is used+ delimiter = None # When this is None, python's csv module's default delimiter is used+ quotechar = None # When this is None, python's csv module's default quotechar is usedheaders = Nonedef process_results(self, response, results):diff --git a/scrapy/utils/console.py b/scrapy/utils/console.pyindex 688e28c34..7eb40f0ce 100644--- a/scrapy/utils/console.py+++ b/scrapy/utils/console.py@@ -47,7 +47,7 @@ def _embed_ptpython_shell(namespace={}, banner=''):def _embed_standard_shell(namespace={}, banner=''):"""Start a standard python shell"""import code- try: # readline module is only available on unix systems+ try: # readline module is only available on unix systemsimport readlineexcept ImportError:pass@@ -72,9 +72,9 @@ def get_shell_embed_func(shells=None, known_shells=None):"""Return the first acceptable shell-embed functionfrom a given list of shell names."""- if shells is None: # list, preference order of shells+ if shells is None: # list, preference order of shellsshells = DEFAULT_PYTHON_SHELLS.keys()- if known_shells is None: # available embeddable shells+ if known_shells is None: # available embeddable shellsknown_shells = DEFAULT_PYTHON_SHELLS.copy()for shell in shells:if shell in known_shells:@@ -97,5 +97,5 @@ def start_python_console(namespace=None, banner='', shells=None):shell = get_shell_embed_func(shells)if shell is not None:shell(namespace=namespace, banner=banner)- except SystemExit: # raised when using exit() in python code.interact+ except SystemExit: # raised when using exit() in python code.interactpassdiff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.pyindex 3fa3ed930..9d3c8fe73 100644--- a/tests/test_command_fetch.py+++ b/tests/test_command_fetch.py@@ -29,6 +29,6 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):@defer.inlineCallbacksdef test_headers(self):_, out, _ = yield self.execute([self.url('/text'), '--headers'])- out = out.replace(b'\r', b'') # required on win32+ out = out.replace(b'\r', b'') # required on win32assert b'Server: TwistedWeb' in out, outassert b'Content-Type: text/plain' in outdiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 60124b93f..2db2417e8 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -349,7 +349,7 @@ class HttpTestCase(unittest.TestCase):return self.download_request(request, Spider('foo')).addCallback(_test)def test_payload(self):- body = b'1'*100 # PayloadResource requires body length to be 100+ body = b'1'*100 # PayloadResource requires body length to be 100request = Request(self.getURL('payload'), method='POST', body=body)d = self.download_request(request, Spider('foo'))d.addCallback(lambda r: r.body)diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 002c4e6bb..537df8d91 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -271,7 +271,6 @@ class EngineTest(unittest.TestCase):self.run.signals_catched[signals.spider_opened])self.assertEqual({'spider': self.run.spider},self.run.signals_catched[signals.spider_idle])- self.run.signals_catched[signals.spider_closed].pop('spider_stats', None) # XXX: remove for scrapy 0.17self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},self.run.signals_catched[signals.spider_closed])diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex a98aa1e6f..57e7b457d 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -149,7 +149,7 @@ class RequestTest(unittest.TestCase):r2 = self.request_class(url="http://www.example.com/", body=b"")assert isinstance(r2.body, bytes)- self.assertEqual(r2.encoding, 'utf-8') # default encoding+ self.assertEqual(r2.encoding, 'utf-8') # default encodingr3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')assert isinstance(r3.body, bytes)diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 0d23f51cc..1fcc5799e 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -238,10 +238,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)])m = self.pipe._mockcalled# only once- self.assertEqual(m[0], 'get_media_requests') # first hook called+ self.assertEqual(m[0], 'get_media_requests') # first hook calledself.assertEqual(m.count('get_media_requests'), 1)self.assertEqual(m.count('item_completed'), 1)- self.assertEqual(m[-1], 'item_completed') # last hook called+ self.assertEqual(m[-1], 'item_completed') # last hook called# twice, one per requestself.assertEqual(m.count('media_to_download'), 2)# one to handle success and other for failure@@ -252,7 +252,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):def test_get_media_requests(self):# returns single Request (without callback)req = Request('http://url')- item = dict(requests=req) # pass a single item+ item = dict(requests=req) # pass a single itemnew_item = yield self.pipe.process_item(item, self.spider)assert new_item is itemassert request_fingerprint(req) in self.info.downloadeddiff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex ecec6135d..7cc17600c 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -548,8 +548,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):(301, 'http://scrapytest.org/3'),(301, 'http://scrapytest.org/4'),),- b'http://scrapytest.org/1', # expected initial referer- b'http://scrapytest.org/1', # expected referer for the redirection request+ b'http://scrapytest.org/1', # expected initial referer+ b'http://scrapytest.org/1', # expected referer for the redirection request),( 'https://scrapytest.org/1','https://scrapytest.org/2',@@ -609,8 +609,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):(301, 'http://scrapytest.org/3'),(301, 'http://scrapytest.org/4'),),- None, # expected initial "Referer"- None, # expected "Referer" for the redirection request+ None, # expected initial "Referer"+ None, # expected "Referer" for the redirection request),( 'https://scrapytest.org/1','https://scrapytest.org/2',@@ -648,8 +648,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):(301, 'http://scrapytest.org/103'),(301, 'http://scrapytest.org/104'),),- b'http://scrapytest.org/101', # expected initial "Referer"- b'http://scrapytest.org/101', # expected referer for the redirection request+ b'http://scrapytest.org/101', # expected initial "Referer"+ b'http://scrapytest.org/101', # expected referer for the redirection request),( 'https://scrapytest.org/201','https://scrapytest.org/202',@@ -757,8 +757,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):(301, 'http://scrapytest.org/103'),(301, 'http://scrapytest.org/104'),),- b'http://scrapytest.org/101', # expected initial referer- b'http://scrapytest.org/101', # expected referer for the redirection request+ b'http://scrapytest.org/101', # expected initial referer+ b'http://scrapytest.org/101', # expected referer for the redirection request),( 'https://scrapytest.org/201','https://scrapytest.org/202',@@ -827,8 +827,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):(301, 'http://scrapytest.org/103'),(301, 'http://scrapytest.org/104'),),- b'http://scrapytest.org/101', # expected initial referer- b'http://scrapytest.org/101', # expected referer for the redirection request+ b'http://scrapytest.org/101', # expected initial referer+ b'http://scrapytest.org/101', # expected referer for the redirection request),( 'https://scrapytest.org/201','https://scrapytest.org/202',diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 49c2befb5..dfbe71ae2 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -14,8 +14,8 @@ class MustbeDeferredTest(unittest.TestCase):return stepsdfd = mustbe_deferred(_append, 1)- dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred- steps.append(2) # add another value, that should be catched by assertEqual+ dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred+ steps.append(2) # add another value, that should be catched by assertEqualreturn dfddef test_unfired_deferred(self):@@ -27,8 +27,8 @@ class MustbeDeferredTest(unittest.TestCase):return dfddfd = mustbe_deferred(_append, 1)- dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred- steps.append(2) # add another value, that should be catched by assertEqual+ dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred+ steps.append(2) # add another value, that should be catched by assertEqualreturn dfddiff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.pyindex edeeacc80..ee7d17062 100644--- a/tests/test_utils_spider.py+++ b/tests/test_utils_spider.py@@ -1,20 +1,16 @@import unittest++from scrapy import Spiderfrom scrapy.http import Requestfrom scrapy.item import BaseItemfrom scrapy.utils.spider import iterate_spider_output, iter_spider_classes-from scrapy.spiders import CrawlSpider---class MyBaseSpider(CrawlSpider):- pass # abstract spider--class MySpider1(MyBaseSpider):+class MySpider1(Spider):name = 'myspider1'-class MySpider2(MyBaseSpider):+class MySpider2(Spider):name = 'myspider2'@@ -35,5 +31,6 @@ class UtilsSpidersTestCase(unittest.TestCase):it = iter_spider_classes(tests.test_utils_spider)self.assertEqual(set(it), {MySpider1, MySpider2})+if __name__ == "__main__":unittest.main()