Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

PEP8 adjustments

Eugenio Lacuestab6bbb2819707
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 4d7d9bac4..ab62c678c 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,6 +1,5 @@
import logging
import signal
-import sys
import warnings
from twisted.internet import reactor, defer
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index f2999daea..e31ab1780 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats.
import csv
import io
-import sys
import pprint
import marshal
import warnings
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index c39de0b52..0903fd4f8 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -165,7 +165,7 @@ class WrappedRequest(object):
def get_header(self, name, default=None):
return to_unicode(self.request.headers.get(name, default),
- errors='replace')
+ errors='replace')
def header_items(self):
return [
diff --git a/scrapy/link.py b/scrapy/link.py
index be1888ef0..a809c5ca4 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -4,6 +4,8 @@ This module defines the Link object used in Link extractors.
For actual link extractors implementation see scrapy.linkextractors, or
its documentation in: docs/topics/link-extractors.rst
"""
+
+
class Link(object):
"""Link objects represent an extracted link by the LinkExtractor."""
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 70f11466b..0d23f51cc 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,5 +1,3 @@
-import sys
-
from testfixtures import LogCapture
from twisted.trial import unittest
from twisted.python.failure import Failure
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index bf56136b1..4147dc944 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -28,17 +28,24 @@ from mitmproxy.tools.main import mitmdump
sys.argv[0] = "mitmdump"
sys.exit(mitmdump())
"""
- cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
- 'keys', 'mitmproxy-ca.pem')
- self.proc = Popen([sys.executable,
- '-c', script,
- '--listen-host', '127.0.0.1',
- '--listen-port', '0',
- '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
- '--certs', cert_path,
- '--ssl-insecure',
- ],
- stdout=PIPE, env=get_testenv())
+ cert_path = os.path.join(
+ os.path.abspath(os.path.dirname(__file__)),
+ 'keys',
+ 'mitmproxy-ca.pem'
+ )
+ self.proc = Popen(
+ [
+ sys.executable,
+ '-c', script,
+ '--listen-host', '127.0.0.1',
+ '--listen-port', '0',
+ '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
+ '--certs', cert_path,
+ '--ssl-insecure',
+ ],
+ stdout=PIPE,
+ env=get_testenv()
+ )
line = self.proc.stdout.readline().decode('utf-8')
host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
@@ -75,9 +82,9 @@ class ProxyConnectTestCase(TestCase):
@defer.inlineCallbacks
def test_https_connect_tunnel(self):
crawler = get_crawler(SimpleSpider)
- with LogCapture() as l:
+ with LogCapture() as logs:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
- self._assert_got_response_code(200, l)
+ self._assert_got_response_code(200, logs)
@pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
@defer.inlineCallbacks
@@ -85,35 +92,35 @@ class ProxyConnectTestCase(TestCase):
proxy = os.environ['https_proxy']
os.environ['https_proxy'] = proxy + '?noconnect'
crawler = get_crawler(SimpleSpider)
- with LogCapture() as l:
+ with LogCapture() as logs:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
- self._assert_got_response_code(200, l)
+ self._assert_got_response_code(200, logs)
@pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
@defer.inlineCallbacks
def test_https_connect_tunnel_error(self):
crawler = get_crawler(SimpleSpider)
- with LogCapture() as l:
+ with LogCapture() as logs:
yield crawler.crawl("https://localhost:99999/status?n=200")
- self._assert_got_tunnel_error(l)
+ self._assert_got_tunnel_error(logs)
@defer.inlineCallbacks
def test_https_tunnel_auth_error(self):
os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
crawler = get_crawler(SimpleSpider)
- with LogCapture() as l:
+ with LogCapture() as logs:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
# The proxy returns a 407 error code but it does not reach the client;
# he just sees a TunnelError.
- self._assert_got_tunnel_error(l)
+ self._assert_got_tunnel_error(logs)
@defer.inlineCallbacks
def test_https_tunnel_without_leak_proxy_authorization_header(self):
request = Request(self.mockserver.url("/echo", is_secure=True))
crawler = get_crawler(SingleRequestSpider)
- with LogCapture() as l:
+ with LogCapture() as logs:
yield crawler.crawl(seed=request)
- self._assert_got_response_code(200, l)
+ self._assert_got_response_code(200, logs)
echo = json.loads(crawler.spider.meta['responses'][0].text)
self.assertTrue('Proxy-Authorization' not in echo['headers'])
@@ -122,9 +129,9 @@ class ProxyConnectTestCase(TestCase):
def test_https_noconnect_auth_error(self):
os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
crawler = get_crawler(SimpleSpider)
- with LogCapture() as l:
+ with LogCapture() as logs:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
- self._assert_got_response_code(407, l)
+ self._assert_got_response_code(407, logs)
def _assert_got_response_code(self, code, log):
print(log)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index faf0d4b73..b36c2a5e3 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -7,7 +7,7 @@ from itertools import count
from scrapy.utils.python import (
memoizemethod_noargs, binary_is_text, equal_attributes,
- WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
+ WeakKeyCache, get_func_args, to_bytes, to_unicode,
without_none_values, MutableChain)