Scrapy, a fast high-level web crawling & scraping framework for Python.
PEP8 adjustments
Eugenio Lacuestab6bbb2819707
diff --git a/scrapy/crawler.py b/scrapy/crawler.pyindex 4d7d9bac4..ab62c678c 100644--- a/scrapy/crawler.py+++ b/scrapy/crawler.py@@ -1,6 +1,5 @@import loggingimport signal-import sysimport warningsfrom twisted.internet import reactor, deferdiff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex f2999daea..e31ab1780 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats.import csvimport io-import sysimport pprintimport marshalimport warningsdiff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.pyindex c39de0b52..0903fd4f8 100644--- a/scrapy/http/cookies.py+++ b/scrapy/http/cookies.py@@ -165,7 +165,7 @@ class WrappedRequest(object):def get_header(self, name, default=None):return to_unicode(self.request.headers.get(name, default),- errors='replace')+ errors='replace')def header_items(self):return [diff --git a/scrapy/link.py b/scrapy/link.pyindex be1888ef0..a809c5ca4 100644--- a/scrapy/link.py+++ b/scrapy/link.py@@ -4,6 +4,8 @@ This module defines the Link object used in Link extractors.For actual link extractors implementation see scrapy.linkextractors, orits documentation in: docs/topics/link-extractors.rst"""++class Link(object):"""Link objects represent an extracted link by the LinkExtractor."""diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 70f11466b..0d23f51cc 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -1,5 +1,3 @@-import sys-from testfixtures import LogCapturefrom twisted.trial import unittestfrom twisted.python.failure import Failurediff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pyindex bf56136b1..4147dc944 100644--- a/tests/test_proxy_connect.py+++ b/tests/test_proxy_connect.py@@ -28,17 +28,24 @@ from mitmproxy.tools.main import mitmdumpsys.argv[0] = "mitmdump"sys.exit(mitmdump())"""- cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),- 'keys', 'mitmproxy-ca.pem')- self.proc = Popen([sys.executable,- '-c', script,- '--listen-host', '127.0.0.1',- '--listen-port', '0',- '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),- '--certs', cert_path,- '--ssl-insecure',- ],- stdout=PIPE, env=get_testenv())+ cert_path = os.path.join(+ os.path.abspath(os.path.dirname(__file__)),+ 'keys',+ 'mitmproxy-ca.pem'+ )+ self.proc = Popen(+ [+ sys.executable,+ '-c', script,+ '--listen-host', '127.0.0.1',+ '--listen-port', '0',+ '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),+ '--certs', cert_path,+ '--ssl-insecure',+ ],+ stdout=PIPE,+ env=get_testenv()+ )line = self.proc.stdout.readline().decode('utf-8')host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)@@ -75,9 +82,9 @@ class ProxyConnectTestCase(TestCase):@defer.inlineCallbacksdef test_https_connect_tunnel(self):crawler = get_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as logs:yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))- self._assert_got_response_code(200, l)+ self._assert_got_response_code(200, logs)@pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')@defer.inlineCallbacks@@ -85,35 +92,35 @@ class ProxyConnectTestCase(TestCase):proxy = os.environ['https_proxy']os.environ['https_proxy'] = proxy + '?noconnect'crawler = get_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as logs:yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))- self._assert_got_response_code(200, l)+ self._assert_got_response_code(200, logs)@pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)@defer.inlineCallbacksdef test_https_connect_tunnel_error(self):crawler = get_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as logs:yield crawler.crawl("https://localhost:99999/status?n=200")- self._assert_got_tunnel_error(l)+ self._assert_got_tunnel_error(logs)@defer.inlineCallbacksdef test_https_tunnel_auth_error(self):os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])crawler = get_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as logs:yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))# The proxy returns a 407 error code but it does not reach the client;# he just sees a TunnelError.- self._assert_got_tunnel_error(l)+ self._assert_got_tunnel_error(logs)@defer.inlineCallbacksdef test_https_tunnel_without_leak_proxy_authorization_header(self):request = Request(self.mockserver.url("/echo", is_secure=True))crawler = get_crawler(SingleRequestSpider)- with LogCapture() as l:+ with LogCapture() as logs:yield crawler.crawl(seed=request)- self._assert_got_response_code(200, l)+ self._assert_got_response_code(200, logs)echo = json.loads(crawler.spider.meta['responses'][0].text)self.assertTrue('Proxy-Authorization' not in echo['headers'])@@ -122,9 +129,9 @@ class ProxyConnectTestCase(TestCase):def test_https_noconnect_auth_error(self):os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'crawler = get_crawler(SimpleSpider)- with LogCapture() as l:+ with LogCapture() as logs:yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))- self._assert_got_response_code(407, l)+ self._assert_got_response_code(407, logs)def _assert_got_response_code(self, code, log):print(log)diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex faf0d4b73..b36c2a5e3 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -7,7 +7,7 @@ from itertools import countfrom scrapy.utils.python import (memoizemethod_noargs, binary_is_text, equal_attributes,- WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,+ WeakKeyCache, get_func_args, to_bytes, to_unicode,without_none_values, MutableChain)