Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove unused imports
Adrián Chavesa2bf340bab79
diff --git a/pytest.ini b/pytest.iniindex 7be5d8572..0b79860f7 100644--- a/pytest.ini+++ b/pytest.ini@@ -21,12 +21,17 @@ addopts =--ignore=docs/utilstwisted = 1flake8-ignore =+ # Files that are only meant to provide top-level imports are expected not+ # to use any of their imports:+ scrapy/core/downloader/handlers/http.py F401+ scrapy/http/__init__.py F401+ # Issues pending a review:# extrasextras/qps-bench-server.py E261 E501extras/qpsclient.py E501 E261 E501# scrapy/commandsscrapy/commands/__init__.py E128 E501- scrapy/commands/check.py F401 E501+ scrapy/commands/check.py E501scrapy/commands/crawl.py E501scrapy/commands/edit.py E501scrapy/commands/fetch.py E401 E501 E128 E502 E731@@ -37,7 +42,6 @@ flake8-ignore =scrapy/commands/shell.py E128 E501 E502scrapy/commands/startproject.py E502 E127 E501 E128scrapy/commands/version.py E501 E128- scrapy/commands/view.py F401# scrapy/contractsscrapy/contracts/__init__.py E501 W504scrapy/contracts/default.py E502 E128@@ -46,17 +50,16 @@ flake8-ignore =scrapy/core/scheduler.py E501scrapy/core/scraper.py E501 E306 E261 E128 W504scrapy/core/spidermw.py E501 E731 E502 E126 E226- scrapy/core/downloader/__init__.py F401 E501+ scrapy/core/downloader/__init__.py E501scrapy/core/downloader/contextfactory.py E501 E128 E126scrapy/core/downloader/middleware.py E501 E502scrapy/core/downloader/tls.py E501 E305 E241scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226scrapy/core/downloader/handlers/__init__.py E501scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127- scrapy/core/downloader/handlers/http.py F401scrapy/core/downloader/handlers/http10.py E501scrapy/core/downloader/handlers/http11.py E501- scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126+ scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126# scrapy/downloadermiddlewaresscrapy/downloadermiddlewares/ajaxcrawl.py E501 E226scrapy/downloadermiddlewares/decompression.py E501@@ -66,19 +69,18 @@ flake8-ignore =scrapy/downloadermiddlewares/httpproxy.py E501scrapy/downloadermiddlewares/redirect.py E501 W504scrapy/downloadermiddlewares/retry.py E501 E126- scrapy/downloadermiddlewares/robotstxt.py F401 E501+ scrapy/downloadermiddlewares/robotstxt.py E501scrapy/downloadermiddlewares/stats.py E501# scrapy/extensionsscrapy/extensions/closespider.py E501 E502 E128 E123scrapy/extensions/corestats.py E501scrapy/extensions/feedexport.py E128 E501- scrapy/extensions/httpcache.py E128 E501 E303 F401+ scrapy/extensions/httpcache.py E128 E501 E303scrapy/extensions/memdebug.py E501scrapy/extensions/spiderstate.py E501scrapy/extensions/telnet.py E501 W504scrapy/extensions/throttle.py E501# scrapy/http- scrapy/http/__init__.py F401scrapy/http/common.py E501scrapy/http/cookies.py E501scrapy/http/request/__init__.py E501@@ -87,7 +89,7 @@ flake8-ignore =scrapy/http/response/__init__.py E501 E128 W293 W291scrapy/http/response/text.py E501 W293 E128 E124# scrapy/linkextractors- scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401+ scrapy/linkextractors/__init__.py E731 E502 E501 E402scrapy/linkextractors/lxmlhtml.py E501 E731 E226# scrapy/loaderscrapy/loader/__init__.py E501 E502 E128@@ -97,8 +99,8 @@ flake8-ignore =scrapy/pipelines/images.py E265 E501scrapy/pipelines/media.py E125 E501 E266# scrapy/selector- scrapy/selector/__init__.py F403 F401- scrapy/selector/unified.py F401 E501 E111+ scrapy/selector/__init__.py F403+ scrapy/selector/unified.py E501 E111# scrapy/settingsscrapy/settings/__init__.py E501scrapy/settings/default_settings.py E501 E261 E114 E116 E226@@ -106,32 +108,30 @@ flake8-ignore =# scrapy/spidermiddlewaresscrapy/spidermiddlewares/httperror.py E501scrapy/spidermiddlewares/offsite.py E501- scrapy/spidermiddlewares/referer.py F401 E501 E129 W503 W504+ scrapy/spidermiddlewares/referer.py E501 E129 W503 W504scrapy/spidermiddlewares/urllength.py E501# scrapy/spiders- scrapy/spiders/__init__.py F401 E501 E402+ scrapy/spiders/__init__.py E501 E402scrapy/spiders/crawl.py E501scrapy/spiders/feed.py E501 E261scrapy/spiders/sitemap.py E501# scrapy/utilsscrapy/utils/benchserver.py E501- scrapy/utils/boto.py F401scrapy/utils/conf.py E402 E502 E501- scrapy/utils/console.py E261 F401 E306 E305- scrapy/utils/curl.py F401+ scrapy/utils/console.py E261 E306 E305scrapy/utils/datatypes.py E501 E226scrapy/utils/decorators.py E501scrapy/utils/defer.py E501 E128scrapy/utils/deprecate.py E128 E501 E127 E502- scrapy/utils/engine.py F401 E261+ scrapy/utils/engine.py E261scrapy/utils/gz.py E305 E501 W504- scrapy/utils/http.py F403 F401 E226+ scrapy/utils/http.py F403 E226scrapy/utils/httpobj.py E501scrapy/utils/iterators.py E501 E701scrapy/utils/log.py E128 W503- scrapy/utils/markup.py F403 F401 W292+ scrapy/utils/markup.py F403 W292scrapy/utils/misc.py E501 E226- scrapy/utils/multipart.py F403 F401 W292+ scrapy/utils/multipart.py F403 W292scrapy/utils/project.py E501scrapy/utils/python.py E501scrapy/utils/reactor.py E226@@ -143,7 +143,7 @@ flake8-ignore =scrapy/utils/spider.py E271 E501scrapy/utils/ssl.py E501scrapy/utils/test.py E501- scrapy/utils/url.py E501 F403 F401 E128 F405+ scrapy/utils/url.py E501 F403 E128 F405# scrapyscrapy/__init__.py E402 E501scrapy/_monkeypatches.py W293@@ -167,29 +167,29 @@ flake8-ignore =scrapy/squeues.py E128scrapy/statscollectors.py E501# tests- tests/__init__.py F401 E402 E501- tests/mockserver.py E401 E501 E126 E123 F401+ tests/__init__.py E402 E501+ tests/mockserver.py E401 E501 E126 E123tests/pipelines.py F841 E226tests/spiders.py E501 E127tests/test_closespider.py E501 E127tests/test_command_fetch.py E501 E261- tests/test_command_parse.py F401 E501 E128 E303 E226+ tests/test_command_parse.py E501 E128 E303 E226tests/test_command_shell.py E501 E128- tests/test_commands.py F401 E128 E501+ tests/test_commands.py E128 E501tests/test_contracts.py E501 E128 W293tests/test_crawl.py E501 E741 E265tests/test_crawler.py F841 E306 E501tests/test_dependencies.py F841 E501 E305- tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123+ tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123tests/test_downloadermiddleware.py E501tests/test_downloadermiddleware_ajaxcrawlable.py E501tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126tests/test_downloadermiddleware_decompression.py E127tests/test_downloadermiddleware_defaultheaders.py E501tests/test_downloadermiddleware_downloadtimeout.py E501- tests/test_downloadermiddleware_httpcache.py E501 E305 F401- tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123- tests/test_downloadermiddleware_httpproxy.py F401 E501 E128+ tests/test_downloadermiddleware_httpcache.py E501 E305+ tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123+ tests/test_downloadermiddleware_httpproxy.py E501 E128tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126tests/test_downloadermiddleware_robotstxt.py E501@@ -197,11 +197,11 @@ flake8-ignore =tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124tests/test_engine.py E401 E501 E502 E128 E261tests/test_exporters.py E501 E731 E306 E128 E124- tests/test_extension_telnet.py F401 F841- tests/test_feedexport.py E501 F401 F841 E241+ tests/test_extension_telnet.py F841+ tests/test_feedexport.py E501 F841 E241tests/test_http_cookies.py E501tests/test_http_headers.py E501- tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123+ tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123tests/test_http_response.py E501 E301 E502 E128 E265tests/test_item.py E701 E128 F841 E306tests/test_link.py E501@@ -211,20 +211,20 @@ flake8-ignore =tests/test_mail.py E128 E501 E305tests/test_middleware.py E501 E128tests/test_pipeline_crawl.py E131 E501 E128 E126- tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226- tests/test_pipeline_images.py F401 F841 E501 E303+ tests/test_pipeline_files.py E501 W293 E303 E272 E226+ tests/test_pipeline_images.py F841 E501 E303tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502tests/test_request_cb_kwargs.py E501tests/test_responsetypes.py E501 E305- tests/test_robotstxt_interface.py F401 E501 W291 E501+ tests/test_robotstxt_interface.py E501 W291 E501tests/test_scheduler.py E501 E126 E123- tests/test_selector.py F401 E501 E127- tests/test_spider.py E501 F401+ tests/test_selector.py E501 E127+ tests/test_spider.py E501tests/test_spidermiddleware.py E501 E226tests/test_spidermiddleware_httperror.py E128 E501 E127 E121tests/test_spidermiddleware_offsite.py E501 E128 E111 W293- tests/test_spidermiddleware_output_chain.py F401 E501 W293 E226- tests/test_spidermiddleware_referer.py F401 E501 F841 E125 E201 E261 E124 E501 E241 E121+ tests/test_spidermiddleware_output_chain.py E501 W293 E226+ tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121tests/test_squeues.py E501 E701 E741tests/test_utils_conf.py E501 E303 E128tests/test_utils_curl.py E501@@ -235,16 +235,16 @@ flake8-ignore =tests/test_utils_iterators.py E501 E128 E129 E303 E241tests/test_utils_log.py E741 E226tests/test_utils_python.py E501 E303 E731 E701 E305- tests/test_utils_reqser.py F401 E501 E128+ tests/test_utils_reqser.py E501 E128tests/test_utils_request.py E501 E128 E305tests/test_utils_response.py E501tests/test_utils_signal.py E741 F841 E731 E226tests/test_utils_sitemap.py E128 E501 E124tests/test_utils_spider.py E261 E305tests/test_utils_template.py E305- tests/test_utils_url.py F401 E501 E127 E305 E211 E125 E501 E226 E241 E126 E123+ tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126tests/test_cmdline/__init__.py E502 E501- tests/test_settings/__init__.py F401 E501 E128+ tests/test_settings/__init__.py E501 E128tests/test_spiderloader/__init__.py E128 E501tests/test_utils_misc/__init__.py E501diff --git a/scrapy/commands/check.py b/scrapy/commands/check.pyindex 3e6c11b7d..9d4437a47 100644--- a/scrapy/commands/check.py+++ b/scrapy/commands/check.py@@ -1,6 +1,4 @@-from __future__ import print_functionimport time-import sysfrom collections import defaultdictfrom unittest import TextTestRunner, TextTestResult as _TextTestResultdiff --git a/scrapy/commands/view.py b/scrapy/commands/view.pyindex 31c17c0ab..41e77ba3b 100644--- a/scrapy/commands/view.py+++ b/scrapy/commands/view.py@@ -1,4 +1,4 @@-from scrapy.commands import fetch, ScrapyCommand+from scrapy.commands import fetchfrom scrapy.utils.response import open_in_browserdiff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 949dacbc8..213268741 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -1,6 +1,4 @@-from __future__ import absolute_importimport random-import warningsfrom time import timefrom datetime import datetimefrom collections import dequediff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.pyindex ac4b867c3..6111e132a 100644--- a/scrapy/core/downloader/handlers/http.py+++ b/scrapy/core/downloader/handlers/http.py@@ -1,3 +1,2 @@-from __future__ import absolute_importfrom .http10 import HTTP10DownloadHandlerfrom .http11 import HTTP11DownloadHandler as HTTPDownloadHandlerdiff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex d8bbdd326..808d1bf21 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -21,7 +21,7 @@ def _get_boto_connection():return http_request.headerstry:- import boto.auth+ import boto.auth # noqa: F401except ImportError:_S3Connection = _v19_S3Connectionelse:diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex f3fabf710..11403957c 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -6,18 +6,16 @@ import osfrom email.utils import mktime_tz, parsedate_tzfrom importlib import import_modulefrom time import time-from warnings import warnfrom weakref import WeakKeyDictionaryfrom six.moves import cPickle as picklefrom w3lib.http import headers_raw_to_dict, headers_dict_to_raw-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Headers, Responsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.project import data_path-from scrapy.utils.python import to_bytes, to_unicode, garbage_collect+from scrapy.utils.python import to_bytes, to_unicodefrom scrapy.utils.request import request_fingerprintdiff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.pyindex e4c62f87b..8c3693f04 100644--- a/scrapy/linkextractors/__init__.py+++ b/scrapy/linkextractors/__init__.py@@ -118,4 +118,4 @@ class FilteringLinkExtractor(object):# Top-level imports-from .lxmlhtml import LxmlLinkExtractor as LinkExtractor+from .lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.pyindex 90e96ee92..a9240c1f6 100644--- a/scrapy/selector/__init__.py+++ b/scrapy/selector/__init__.py@@ -1,4 +1,4 @@"""Selectors"""-from scrapy.selector.unified import *+from scrapy.selector.unified import * # noqa: F401diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.pyindex 62fda40b0..a08955dc9 100644--- a/scrapy/selector/unified.py+++ b/scrapy/selector/unified.py@@ -2,12 +2,10 @@XPath selectors based on lxml"""-import warningsfrom parsel import Selector as _ParselSelectorfrom scrapy.utils.trackref import object_reffrom scrapy.utils.python import to_bytesfrom scrapy.http import HtmlResponse, XmlResponse-from scrapy.utils.decorators import deprecated__all__ = ['Selector', 'SelectorList']diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex 94095bc27..8d15dfceb 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -10,7 +10,6 @@ from scrapy import signalsfrom scrapy.http import Requestfrom scrapy.utils.trackref import object_reffrom scrapy.utils.url import url_is_from_spider-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.deprecate import method_is_overridden@@ -100,6 +99,6 @@ class Spider(object_ref):# Top-level imports-from scrapy.spiders.crawl import CrawlSpider, Rule-from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider-from scrapy.spiders.sitemap import SitemapSpider+from scrapy.spiders.crawl import CrawlSpider, Rule # noqa: F401+from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider # noqa: F401+from scrapy.spiders.sitemap import SitemapSpider # noqa: F401diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.pyindex c8fc911bb..b76d5e56e 100644--- a/scrapy/utils/boto.py+++ b/scrapy/utils/boto.py@@ -7,7 +7,7 @@ from scrapy.exceptions import NotConfigureddef is_botocore():try:- import botocore+ import botocore # noqa: F401return Trueexcept ImportError:raise NotConfigured('missing botocore library')diff --git a/scrapy/utils/console.py b/scrapy/utils/console.pyindex a26e84d38..688e28c34 100644--- a/scrapy/utils/console.py+++ b/scrapy/utils/console.py@@ -52,7 +52,7 @@ def _embed_standard_shell(namespace={}, banner=''):except ImportError:passelse:- import rlcompleter+ import rlcompleter # noqa: F401readline.parse_and_bind("tab:complete")@wraps(_embed_standard_shell)def wrapper(namespace=namespace, banner=''):diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.pyindex b3fd0a497..7fb25a71d 100644--- a/scrapy/utils/curl.py+++ b/scrapy/utils/curl.py@@ -4,7 +4,7 @@ from shlex import splitfrom six.moves.http_cookies import SimpleCookiefrom six.moves.urllib.parse import urlparse-from six import string_types, iteritems+from six import iteritemsfrom w3lib.http import basic_auth_headerdiff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.pyindex 2c20b5c88..267c7ecd1 100644--- a/scrapy/utils/engine.py+++ b/scrapy/utils/engine.py@@ -1,7 +1,7 @@"""Some debugging functions for working with the Scrapy engine"""-from __future__ import print_function-from time import time # used in global tests code+# used in global tests code+from time import time # noqa: F401def get_engine_status(engine):diff --git a/scrapy/utils/http.py b/scrapy/utils/http.pyindex ad49ef3e9..bab262393 100644--- a/scrapy/utils/http.py+++ b/scrapy/utils/http.py@@ -8,7 +8,7 @@ import warningsfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.decorators import deprecated-from w3lib.http import *+from w3lib.http import * # noqa: F401warnings.warn("Module `scrapy.utils.http` is deprecated, "diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.pyindex a18f308a3..2455fcc16 100644--- a/scrapy/utils/markup.py+++ b/scrapy/utils/markup.py@@ -6,7 +6,7 @@ For new code, always import from w3lib.html instead of this moduleimport warningsfrom scrapy.exceptions import ScrapyDeprecationWarning-from w3lib.html import *+from w3lib.html import * # noqa: F401warnings.warn("Module `scrapy.utils.markup` is deprecated. "diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.pyindex c2d8afd07..e81f63152 100644--- a/scrapy/utils/multipart.py+++ b/scrapy/utils/multipart.py@@ -6,7 +6,7 @@ For new code, always import from w3lib.form instead of this moduleimport warningsfrom scrapy.exceptions import ScrapyDeprecationWarning-from w3lib.form import *+from w3lib.form import * # noqa: F401warnings.warn("Module `scrapy.utils.multipart` is deprecated. "diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex b3a4be007..a3e1bde63 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -12,7 +12,6 @@ from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse# scrapy.utils.url was moved to w3lib.url and import * ensures this# move doesn't break old codefrom w3lib.url import *-from w3lib.url import _safe_chars, _unquotepathfrom scrapy.utils.python import to_unicodediff --git a/tests/mockserver.py b/tests/mockserver.pyindex b766bb653..7ebb8bb62 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -1,9 +1,11 @@-from __future__ import print_function-import sys, time, random, os, json-from six.moves.urllib.parse import urlencode+import json+import os+import random+import sysfrom subprocess import Popen, PIPEfrom OpenSSL import SSL+from six.moves.urllib.parse import urlencodefrom twisted.web.server import Site, NOT_DONE_YETfrom twisted.web.resource import Resourcefrom twisted.web.static import Filediff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 59d4a3eec..0421529a6 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -543,7 +543,7 @@ class Https11InvalidDNSPattern(Https11TestCase):def setUp(self):try:- from service_identity.exceptions import CertificateError+ from service_identity.exceptions import CertificateError # noqa: F401except ImportError:raise unittest.SkipTest("cryptography lib is too old")self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"'@@ -778,7 +778,7 @@ class S3TestCase(unittest.TestCase):@contextlib.contextmanagerdef _mocked_date(self, date):try:- import botocore.auth+ import botocore.auth # noqa: F401except ImportError:yieldelse:@@ -843,8 +843,10 @@ class S3TestCase(unittest.TestCase):b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')def test_request_signing5(self):- try: import botocore- except ImportError: pass+ try:+ import botocore # noqa: F401+ except ImportError:+ passelse:raise unittest.SkipTest('botocore does not support overriding date with x-amz-date')diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex 00e6c685e..9401dd66d 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -1,12 +1,9 @@-from __future__ import print_functionimport timeimport tempfileimport shutilimport unittestimport email.utilsfrom contextlib import contextmanager-import pytest-import sysfrom scrapy.http import Response, HtmlResponse, Requestfrom scrapy.spiders import Spiderdiff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.pyindex 0745c8dd3..c6a823b53 100644--- a/tests/test_downloadermiddleware_httpcompression.py+++ b/tests/test_downloadermiddleware_httpcompression.py@@ -70,7 +70,7 @@ class HttpCompressionTest(TestCase):def test_process_response_br(self):try:- import brotli+ import brotli # noqa: F401except ImportError:raise SkipTest("no brotli")response = self._getresponse('br')diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex 30920b2da..36743b1de 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -1,11 +1,10 @@import os-import sysfrom functools import partial-from twisted.trial.unittest import TestCase, SkipTest+from twisted.trial.unittest import TestCasefrom scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddlewarefrom scrapy.exceptions import NotConfigured-from scrapy.http import Response, Request+from scrapy.http import Requestfrom scrapy.spiders import Spiderfrom scrapy.crawler import Crawlerfrom scrapy.settings import Settingsdiff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.pyindex 875ceb83c..873a97248 100644--- a/tests/test_extension_telnet.py+++ b/tests/test_extension_telnet.py@@ -3,7 +3,7 @@ from twisted.conch.telnet import ITelnetProtocolfrom twisted.cred import credentialsfrom twisted.internet import defer-from scrapy.extensions.telnet import TelnetConsole, logger+from scrapy.extensions.telnet import TelnetConsolefrom scrapy.utils.test import get_crawlerdiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 87139e81f..ce3c4f059 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -167,7 +167,7 @@ class S3FeedStorageTest(unittest.TestCase):create=True)def test_parse_credentials(self):try:- import boto+ import boto # noqa: F401except ImportError:raise unittest.SkipTest("S3FeedStorage requires boto")aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',@@ -268,7 +268,7 @@ class S3FeedStorageTest(unittest.TestCase):@defer.inlineCallbacksdef test_store_botocore_without_acl(self):try:- import botocore+ import botocore # noqa: F401except ImportError:raise unittest.SkipTest('botocore is required')@@ -288,7 +288,7 @@ class S3FeedStorageTest(unittest.TestCase):@defer.inlineCallbacksdef test_store_botocore_with_acl(self):try:- import botocore+ import botocore # noqa: F401except ImportError:raise unittest.SkipTest('botocore is required')diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex 5134a03b9..9df6ff67b 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -1,5 +1,3 @@-# -*- coding: utf-8 -*--import cgiimport unittestimport reimport json@@ -8,7 +6,7 @@ from urllib.parse import unquote_to_bytesimport warningsfrom six.moves import xmlrpc_client as xmlrpclib-from six.moves.urllib.parse import urlparse, parse_qs, unquote+from six.moves.urllib.parse import urlparse, parse_qsfrom scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponsefrom scrapy.utils.python import to_bytes, to_unicodediff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 4f7265763..7f1cb4a11 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -1,7 +1,6 @@import ioimport hashlibimport random-import warningsfrom tempfile import mkdtempfrom shutil import rmtreediff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex 080507276..27d79437b 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -5,7 +5,7 @@ from twisted.trial import unittestdef reppy_available():# check if reppy parser is installedtry:- from reppy.robots import Robots+ from reppy.robots import Robots # noqa: F401except ImportError:return Falsereturn True@@ -14,7 +14,7 @@ def reppy_available():def rerp_available():# check if robotexclusionrulesparser is installedtry:- from robotexclusionrulesparser import RobotExclusionRulesParser+ from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: F401except ImportError:return Falsereturn True@@ -23,7 +23,7 @@ def rerp_available():def protego_available():# check if protego parser is installedtry:- from protego import Protego+ from protego import Protego # noqa: F401except ImportError:return Falsereturn Truediff --git a/tests/test_selector.py b/tests/test_selector.pyindex b2565dd78..09c2546fb 100644--- a/tests/test_selector.py+++ b/tests/test_selector.py@@ -1,9 +1,9 @@-import warningsimport weakref+from twisted.trial import unittest+from scrapy.http import TextResponse, HtmlResponse, XmlResponsefrom scrapy.selector import Selector-from lxml import etreeclass SelectorTestCase(unittest.TestCase):diff --git a/tests/test_spider.py b/tests/test_spider.pyindex c0fccfdd6..aa43e3b3a 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -15,7 +15,6 @@ from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \CSVFeedSpider, SitemapSpiderfrom scrapy.linkextractors import LinkExtractorfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.trackref import object_reffrom scrapy.utils.test import get_crawlerdiff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 940e31070..5b7b5e7aa 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -6,7 +6,6 @@ from twisted.internet import deferfrom scrapy import Spider, Requestfrom scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServer-from tests.spiders import MockServerSpiderclass LogExceptionMiddleware:diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex a9c31a983..2be6a1cd5 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -2,7 +2,6 @@ from six.moves.urllib.parse import urlparsefrom unittest import TestCaseimport warnings-from scrapy.exceptions import NotConfiguredfrom scrapy.http import Response, Requestfrom scrapy.settings import Settingsfrom scrapy.spiders import Spiderdiff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.pyindex 92cd16de7..06d9c004c 100644--- a/tests/test_utils_reqser.py+++ b/tests/test_utils_reqser.py@@ -1,8 +1,4 @@-# -*- coding: utf-8 -*-import unittest-import sys--import sixfrom scrapy.http import Request, FormRequestfrom scrapy.spiders import Spiderdiff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex 93addc082..c7bcaf88b 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -1,13 +1,9 @@# -*- coding: utf-8 -*-import unittest-import six-from six.moves.urllib.parse import urlparse-from scrapy.spiders import Spiderfrom scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,- add_http_if_no_scheme, guess_scheme,- parse_url, strip_url)+ add_http_if_no_scheme, guess_scheme, strip_url)__doctests__ = ['scrapy.utils.url']diff --git a/tox.ini b/tox.iniindex fd75d18e2..e8672ea27 100644--- a/tox.ini+++ b/tox.ini@@ -73,6 +73,7 @@ commands =basepython = python3.8deps ={[testenv]deps}+ -r docs/requirements.txtpytest-flake8commands =py.test --flake8 {posargs:docs scrapy tests}