Scrapy, a fast high-level web crawling & scraping framework for Python.
Upgrade CI tools
Adrián Chavesb07d3f85a389
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex f5fc1285f..6840bfe23 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -5,14 +5,14 @@ repos:- id: banditargs: [-r, -c, .bandit.yml]- repo: https://github.com/PyCQA/flake8- rev: 6.0.0+ rev: 5.0.4 # 6.0.0 drops Python 3.7 supporthooks:- id: flake8- repo: https://github.com/psf/black.git- rev: 22.12.0+ rev: 23.1.0hooks:- id: black- repo: https://github.com/pycqa/isort- rev: 5.12.0+ rev: 5.11.5 # 5.12 drops Python 3.7 supporthooks:- id: isortdiff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.pyindex efb4c202e..1f270837c 100644--- a/docs/utils/linkfix.py+++ b/docs/utils/linkfix.py@@ -18,7 +18,6 @@ from pathlib import Pathdef main():-# Used for remembering the file (and its contents)# so we don't have to open the same file again._filename = None@@ -50,7 +49,6 @@ def main():else:# If this is a new fileif newfilename != _filename:-# Update the previous fileif _filename:Path(_filename).write_text(_contents, encoding="utf-8")diff --git a/extras/qpsclient.py b/extras/qpsclient.pyindex acad71e07..119dfdabb 100644--- a/extras/qpsclient.py+++ b/extras/qpsclient.py@@ -13,7 +13,6 @@ from scrapy.spiders import Spiderclass QPSSpider(Spider):-name = "qps"benchurl = "http://localhost:8880/"diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex de68c43a5..9baee3a48 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -14,7 +14,6 @@ from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cliclass ScrapyCommand:-requires_project = Falsecrawler_process: Optional[CrawlerProcess] = Nonediff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.pyindex 911e5afe6..e1ccdc451 100644--- a/scrapy/commands/bench.py+++ b/scrapy/commands/bench.py@@ -9,7 +9,6 @@ from scrapy.linkextractors import LinkExtractorclass Command(ScrapyCommand):-default_settings = {"LOG_LEVEL": "INFO","LOGSTATS_INTERVAL": 1,diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.pyindex df8006f36..2f0f1c7b9 100644--- a/scrapy/commands/crawl.py+++ b/scrapy/commands/crawl.py@@ -3,7 +3,6 @@ from scrapy.exceptions import UsageErrorclass Command(BaseRunSpiderCommand):-requires_project = Truedef syntax(self):diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.pyindex ca591011c..03a8ed5c7 100644--- a/scrapy/commands/edit.py+++ b/scrapy/commands/edit.py@@ -6,7 +6,6 @@ from scrapy.exceptions import UsageErrorclass Command(ScrapyCommand):-requires_project = Truedefault_settings = {"LOG_ENABLED": False}diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.pyindex a9076c5b1..1359e445f 100644--- a/scrapy/commands/fetch.py+++ b/scrapy/commands/fetch.py@@ -10,7 +10,6 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_requestclass Command(ScrapyCommand):-requires_project = Falsedef syntax(self):diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex 90dd0874e..c1565a138 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -32,7 +32,6 @@ def extract_domain(url):class Command(ScrapyCommand):-requires_project = Falsedefault_settings = {"LOG_ENABLED": False}diff --git a/scrapy/commands/list.py b/scrapy/commands/list.pyindex 2a81bd182..2f5032360 100644--- a/scrapy/commands/list.py+++ b/scrapy/commands/list.py@@ -2,7 +2,6 @@ from scrapy.commands import ScrapyCommandclass Command(ScrapyCommand):-requires_project = Truedefault_settings = {"LOG_ENABLED": False}diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.pyindex 8a75f9270..58ed89a81 100644--- a/scrapy/commands/runspider.py+++ b/scrapy/commands/runspider.py@@ -24,7 +24,6 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType:class Command(BaseRunSpiderCommand):-requires_project = Falsedefault_settings = {"SPIDER_LOADER_WARN_ONLY": True}diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.pyindex c43298ac7..318187204 100644--- a/scrapy/commands/settings.py+++ b/scrapy/commands/settings.py@@ -5,7 +5,6 @@ from scrapy.settings import BaseSettingsclass Command(ScrapyCommand):-requires_project = Falsedefault_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex 05c76d1eb..63c23d04c 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -13,7 +13,6 @@ from scrapy.utils.url import guess_schemeclass Command(ScrapyCommand):-requires_project = Falsedefault_settings = {"KEEP_ALIVE": True,diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.pyindex 88bd5bb33..fde609c6f 100644--- a/scrapy/commands/startproject.py+++ b/scrapy/commands/startproject.py@@ -28,7 +28,6 @@ def _make_writable(path):class Command(ScrapyCommand):-requires_project = Falsedefault_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}diff --git a/scrapy/commands/version.py b/scrapy/commands/version.pyindex faa0e242c..47582866b 100644--- a/scrapy/commands/version.py+++ b/scrapy/commands/version.py@@ -4,7 +4,6 @@ from scrapy.utils.versions import scrapy_components_versionsclass Command(ScrapyCommand):-default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}def syntax(self):diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 7e0b62bb0..7d27a5193 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -69,7 +69,6 @@ def _get_concurrency_delay(concurrency, spider, settings):class Downloader:-DOWNLOAD_SLOT = "download_slot"def __init__(self, crawler):diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 8de5459e9..c3704de3d 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -292,7 +292,6 @@ class ScrapyProxyAgent(Agent):class ScrapyAgent:-_Agent = Agent_ProxyAgent = ScrapyProxyAgent_TunnelingAgent = TunnelingAgentdiff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.pyindex 5a94e66a6..56df48b2e 100644--- a/scrapy/core/downloader/middleware.py+++ b/scrapy/core/downloader/middleware.py@@ -17,7 +17,6 @@ from scrapy.utils.defer import deferred_from_coro, mustbe_deferredclass DownloaderMiddlewareManager(MiddlewareManager):-component_name = "downloader middleware"@classmethoddiff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.pyindex 4558402b2..3d103652b 100644--- a/scrapy/core/downloader/webclient.py+++ b/scrapy/core/downloader/webclient.py@@ -40,7 +40,6 @@ def _parse(url):class ScrapyHTTPPageGetter(HTTPClient):-delimiter = b"\n"def connectionMade(self):@@ -103,7 +102,6 @@ class ScrapyHTTPPageGetter(HTTPClient):# Twisted (https://github.com/twisted/twisted/pull/643), we merged its# non-overridden code into this class.class ScrapyHTTPClientFactory(ClientFactory):-protocol = ScrapyHTTPPageGetterwaiting = 1diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.pyindex ba9c37e38..971cf52e4 100644--- a/scrapy/core/spidermw.py+++ b/scrapy/core/spidermw.py@@ -46,7 +46,6 @@ def _isiterable(o) -> bool:class SpiderMiddlewareManager(MiddlewareManager):-component_name = "spider middleware"def __init__(self, *middlewares):diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.pyindex 137ed5b18..04ae719de 100644--- a/scrapy/downloadermiddlewares/ajaxcrawl.py+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py@@ -30,7 +30,6 @@ class AjaxCrawlMiddleware:return cls(crawler.settings)def process_response(self, request, response, spider):-if not isinstance(response, HtmlResponse) or response.status != 200:return responsediff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.pyindex 74c55f6e2..b9316c43a 100644--- a/scrapy/downloadermiddlewares/httpcache.py+++ b/scrapy/downloadermiddlewares/httpcache.py@@ -27,7 +27,6 @@ HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddlewclass HttpCacheMiddleware:-DOWNLOAD_EXCEPTIONS = (defer.TimeoutError,TimeoutError,diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.pyindex 7b5f4824a..ead426951 100644--- a/scrapy/downloadermiddlewares/httpcompression.py+++ b/scrapy/downloadermiddlewares/httpcompression.py@@ -53,7 +53,6 @@ class HttpCompressionMiddleware:request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))def process_response(self, request, response, spider):-if request.method == "HEAD":return responseif isinstance(response, Response):diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex f442a3012..65f1d2224 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -26,7 +26,6 @@ def _build_redirect_request(source_request, *, url, **kwargs):class BaseRedirectMiddleware:-enabled_setting = "REDIRECT_ENABLED"def __init__(self, settings):@@ -115,7 +114,6 @@ class RedirectMiddleware(BaseRedirectMiddleware):class MetaRefreshMiddleware(BaseRedirectMiddleware):-enabled_setting = "METAREFRESH_ENABLED"def __init__(self, settings):diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex 11a30911c..081642a4b 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -122,7 +122,6 @@ def get_retry_request(class RetryMiddleware:-# IOError is raised by the HttpCompression middleware when trying to# decompress an empty responseEXCEPTIONS_TO_RETRY = (diff --git a/scrapy/extension.py b/scrapy/extension.pyindex 89f6a694d..4e365cfa1 100644--- a/scrapy/extension.py+++ b/scrapy/extension.py@@ -8,7 +8,6 @@ from scrapy.utils.conf import build_component_listclass ExtensionManager(MiddlewareManager):-component_name = "extension"@classmethoddiff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.pyindex 2d120a6ed..dfe843974 100644--- a/scrapy/extensions/httpcache.py+++ b/scrapy/extensions/httpcache.py@@ -41,7 +41,6 @@ class DummyPolicy:class RFC2616Policy:-MAXAGE = 3600 * 24 * 365 # one yeardef __init__(self, settings):diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.pyindex 35e2808c2..510c903db 100644--- a/scrapy/http/request/json_request.py+++ b/scrapy/http/request/json_request.py@@ -15,7 +15,6 @@ from scrapy.utils.deprecate import create_deprecated_classclass JsonRequest(Request):-attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None:diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex e45d95602..73bb811de 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -29,7 +29,6 @@ _NONE = object()class TextResponse(Response):-_DEFAULT_ENCODING = "ascii"_cached_decoded_json = _NONEdiff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 66cf58d89..dd8dcdf7c 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -194,7 +194,6 @@ class LxmlLinkExtractor:return Truedef matches(self, url):-if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):return Falseif self.deny_domains and url_is_from_any_domain(url, self.deny_domains):diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.pyindex 536341fc6..df574a0a1 100644--- a/scrapy/pipelines/__init__.py+++ b/scrapy/pipelines/__init__.py@@ -10,7 +10,6 @@ from scrapy.utils.defer import deferred_f_from_coro_fclass ItemPipelineManager(MiddlewareManager):-component_name = "item pipeline"@classmethoddiff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 3430603ab..0be28933e 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -187,7 +187,6 @@ class S3FilesStore:class GCSFilesStore:-GCS_PROJECT_ID = NoneCACHE_CONTROL = "max-age=172800"@@ -253,7 +252,6 @@ class GCSFilesStore:class FTPFilesStore:-FTP_USERNAME = NoneFTP_PASSWORD = NoneUSE_ACTIVE_MODE = Nonediff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 31f51c004..401029439 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -23,7 +23,6 @@ def _DUMMY_CALLBACK(response):class MediaPipeline:-LOG_FAILED_RESULTS = Trueclass SpiderInfo:diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.pyindex 6af8915c2..f01e9096c 100644--- a/scrapy/responsetypes.py+++ b/scrapy/responsetypes.py@@ -12,7 +12,6 @@ from scrapy.utils.python import binary_is_text, to_bytes, to_unicodeclass ResponseTypes:-CLASSES = {"text/html": "scrapy.http.HtmlResponse","application/atom+xml": "scrapy.http.XmlResponse",diff --git a/scrapy/shell.py b/scrapy/shell.pyindex ae6e641fd..bb3b1461c 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -25,7 +25,6 @@ from scrapy.utils.response import open_in_browserclass Shell:-relevant_classes = (Crawler, Spider, Request, Response, Settings)def __init__(self, crawler, update_vars=None, code=None):diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex d86f55a40..fd91e658b 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -34,7 +34,6 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default"class ReferrerPolicy:-NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMESname: strdiff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex 05c425948..31e845716 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -60,7 +60,6 @@ class Rule:class CrawlSpider(Spider):-rules: Sequence[Rule] = ()def __init__(self, *a, **kw):diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.pyindex 5ec0504a8..6afadc577 100644--- a/scrapy/spiders/feed.py+++ b/scrapy/spiders/feed.py@@ -89,7 +89,7 @@ class XMLFeedSpider(Spider):yield nodedef _register_namespaces(self, selector):- for (prefix, uri) in self.namespaces:+ for prefix, uri in self.namespaces:selector.register_namespace(prefix, uri)diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex c3cca9699..aaf75a519 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -10,7 +10,6 @@ logger = logging.getLogger(__name__)class SitemapSpider(Spider):-sitemap_urls = ()sitemap_rules = [("", "parse")]sitemap_follow = [""]diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.pyindex 1089ba7b8..38884a9f0 100644--- a/scrapy/utils/benchserver.py+++ b/scrapy/utils/benchserver.py@@ -6,7 +6,6 @@ from twisted.web.server import Siteclass Root(Resource):-isLeaf = Truedef getChild(self, name, request):diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.pyindex 10497e9ed..fa57a4f26 100644--- a/scrapy/utils/datatypes.py+++ b/scrapy/utils/datatypes.py@@ -11,7 +11,6 @@ from collections.abc import Mappingclass CaselessDict(dict):-__slots__ = ()def __init__(self, seq=None):diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.pyindex 61a4347ea..f4d6e0451 100644--- a/scrapy/utils/deprecate.py+++ b/scrapy/utils/deprecate.py@@ -54,7 +54,6 @@ def create_deprecated_class("""class DeprecatedClass(new_class.__class__):-deprecated_class = Nonewarned_on_subclass = Falsediff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.pyindex 358f41679..414658944 100644--- a/scrapy/utils/serialize.py+++ b/scrapy/utils/serialize.py@@ -9,7 +9,6 @@ from scrapy.http import Request, Responseclass ScrapyJSONEncoder(json.JSONEncoder):-DATE_FORMAT = "%Y-%m-%d"TIME_FORMAT = "%H:%M:%S"diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.pyindex ecb2e31bf..5f9bdef37 100644--- a/scrapy/utils/testproc.py+++ b/scrapy/utils/testproc.py@@ -5,7 +5,6 @@ from twisted.internet import defer, protocolclass ProcessTest:-command = Noneprefix = [sys.executable, "-m", "scrapy.cmdline"]cwd = os.getcwd() # trial chdirs to temp dirdiff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex 0a27ccd6d..833aa3e20 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -144,7 +144,6 @@ def strip_url(origin_only=False,strip_fragment=True,):-"""Strip URL string from some of its components:- ``strip_credentials`` removes "user:password@"diff --git a/tests/mockserver.py b/tests/mockserver.pyindex 7991da9dc..d541f3f7c 100644--- a/tests/mockserver.py+++ b/tests/mockserver.py@@ -95,7 +95,6 @@ class BrokenDownloadResource(resource.Resource):class LeafResource(resource.Resource):-isLeaf = Truedef deferRequest(self, request, delay, f, *a, **kw):diff --git a/tests/requirements.txt b/tests/requirements.txtindex d9373dfa8..618949795 100644--- a/tests/requirements.txt+++ b/tests/requirements.txt@@ -2,7 +2,7 @@attrspyftpdlibpytest-pytest-cov==3.0.0+pytest-cov==4.0.0pytest-xdistsybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422testfixturesdiff --git a/tests/spiders.py b/tests/spiders.pyindex 2036e8cf3..6ff48f471 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -25,7 +25,6 @@ class MockServerSpider(Spider):class MetaSpider(MockServerSpider):-name = "meta"def __init__(self, *args, **kwargs):@@ -37,7 +36,6 @@ class MetaSpider(MockServerSpider):class FollowAllSpider(MetaSpider):-name = "follow"link_extractor = LinkExtractor()@@ -59,7 +57,6 @@ class FollowAllSpider(MetaSpider):class DelaySpider(MetaSpider):-name = "delay"def __init__(self, n=1, b=0, *args, **kwargs):@@ -81,7 +78,6 @@ class DelaySpider(MetaSpider):class SimpleSpider(MetaSpider):-name = "simple"def __init__(self, url="http://localhost:8998", *args, **kwargs):@@ -93,7 +89,6 @@ class SimpleSpider(MetaSpider):class AsyncDefSpider(SimpleSpider):-name = "asyncdef"async def parse(self, response):@@ -102,7 +97,6 @@ class AsyncDefSpider(SimpleSpider):class AsyncDefAsyncioSpider(SimpleSpider):-name = "asyncdef_asyncio"async def parse(self, response):@@ -112,7 +106,6 @@ class AsyncDefAsyncioSpider(SimpleSpider):class AsyncDefAsyncioReturnSpider(SimpleSpider):-name = "asyncdef_asyncio_return"async def parse(self, response):@@ -123,7 +116,6 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):-name = "asyncdef_asyncio_return_single_element"async def parse(self, response):@@ -134,7 +126,6 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):-name = "asyncdef_asyncio_reqs_return"async def parse(self, response):@@ -191,7 +182,6 @@ class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):class AsyncDefAsyncioGenSpider(SimpleSpider):-name = "asyncdef_asyncio_gen"async def parse(self, response):@@ -201,7 +191,6 @@ class AsyncDefAsyncioGenSpider(SimpleSpider):class AsyncDefAsyncioGenLoopSpider(SimpleSpider):-name = "asyncdef_asyncio_gen_loop"async def parse(self, response):@@ -212,7 +201,6 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider):class AsyncDefAsyncioGenComplexSpider(SimpleSpider):-name = "asyncdef_asyncio_gen_complex"initial_reqs = 4following_reqs = 3@@ -246,7 +234,6 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):class ItemSpider(FollowAllSpider):-name = "item"def parse(self, response):@@ -261,7 +248,6 @@ class DefaultError(Exception):class ErrorSpider(FollowAllSpider):-name = "error"exception_cls = DefaultError@@ -275,7 +261,6 @@ class ErrorSpider(FollowAllSpider):class BrokenStartRequestsSpider(FollowAllSpider):-fail_before_yield = Falsefail_yielding = False@@ -305,7 +290,6 @@ class BrokenStartRequestsSpider(FollowAllSpider):class SingleRequestSpider(MetaSpider):-seed = Nonecallback_func = Noneerrback_func = None@@ -451,7 +435,6 @@ class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseclass BytesReceivedCallbackSpider(MetaSpider):-full_response_length = 2**18@classmethoddiff --git a/tests/test_command_check.py b/tests/test_command_check.pyindex 1f299587f..129ef0121 100644--- a/tests/test_command_check.py+++ b/tests/test_command_check.py@@ -2,7 +2,6 @@ from tests.test_commands import CommandTestclass CheckCommandTest(CommandTest):-command = "check"def setUp(self):diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.pyindex 124c968c2..d2027d1c2 100644--- a/tests/test_command_fetch.py+++ b/tests/test_command_fetch.py@@ -6,7 +6,6 @@ from scrapy.utils.testsite import SiteTestclass FetchTest(ProcessTest, SiteTest, unittest.TestCase):-command = "fetch"@defer.inlineCallbacksdiff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex 8ce82db86..6589381f3 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -9,7 +9,6 @@ from tests import NON_EXISTING_RESOLVABLE, tests_datadirclass ShellTest(ProcessTest, SiteTest, unittest.TestCase):-command = "shell"@defer.inlineCallbacksdiff --git a/tests/test_command_version.py b/tests/test_command_version.pyindex 3bf6019b5..a52d0d13c 100644--- a/tests/test_command_version.py+++ b/tests/test_command_version.py@@ -8,7 +8,6 @@ from scrapy.utils.testproc import ProcessTestclass VersionTest(ProcessTest, unittest.TestCase):-command = "version"@defer.inlineCallbacksdiff --git a/tests/test_commands.py b/tests/test_commands.pyindex d7ee75559..00ddcdd3e 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -223,7 +223,6 @@ def get_permissions_dict(class StartprojectTemplatesTest(ProjectTest):-maxDiff = Nonedef setUp(self):@@ -604,7 +603,6 @@ class MiscCommandsTest(CommandTest):class RunSpiderCommandTest(CommandTest):-spider_filename = "myspider.py"debug_log_spider = """@@ -873,7 +871,6 @@ class MySpider(scrapy.Spider):@skipIf(platform.system() != "Windows", "Windows required for .pyw files")class WindowsRunSpiderCommandTest(RunSpiderCommandTest):-spider_filename = "myspider.pyw"def setUp(self):diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.pyindex 8090d50b9..31fa1430d 100644--- a/tests/test_downloader_handlers_http2.py+++ b/tests/test_downloader_handlers_http2.py@@ -25,7 +25,6 @@ from tests.test_downloader_handlers import (@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")class Https2TestCase(Https11TestCase):-scheme = "https"HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex 2be32e37b..060cfe08b 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -16,7 +16,6 @@ from scrapy.utils.test import get_crawler, get_from_asyncio_queueclass ManagerTestCase(TestCase):-settings_dict = Nonedef setUp(self):diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.pyindex 412c20a78..95739414e 100644--- a/tests/test_downloadermiddleware_decompression.py+++ b/tests/test_downloadermiddleware_decompression.py@@ -17,7 +17,6 @@ def _test_data(formats):class DecompressionMiddlewareTest(TestCase):-test_formats = ["tar", "xml.bz2", "xml.gz", "zip"]uncompressed_body, test_responses = _test_data(test_formats)diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex a355a9b5b..f80eff3e6 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -14,7 +14,6 @@ from scrapy.utils.test import get_crawlerclass _BaseTest(unittest.TestCase):-storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"policy_class = "scrapy.extensions.httpcache.RFC2616Policy"@@ -146,12 +145,10 @@ class DefaultStorageTest(_BaseTest):class DbmStorageTest(DefaultStorageTest):-storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):-dbm_module = "tests.mocks.dummydbm"def _get_settings(self, **new_settings):@@ -165,7 +162,6 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):class FilesystemStorageTest(DefaultStorageTest):-storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"@@ -176,7 +172,6 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):class DummyPolicyTest(_BaseTest):-policy_class = "scrapy.extensions.httpcache.DummyPolicy"def test_middleware(self):@@ -270,7 +265,6 @@ class DummyPolicyTest(_BaseTest):class RFC2616PolicyTest(DefaultStorageTest):-policy_class = "scrapy.extensions.httpcache.RFC2616Policy"def _process_requestresponse(self, mw, request, response):diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex ca125ba36..9ec51f75b 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -13,7 +13,6 @@ spider = Spider("foo")class TestHttpProxyMiddleware(TestCase):-failureException = AssertionErrordef setUp(self):diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex 02854c2a7..63bd61848 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -127,7 +127,6 @@ class RetryTest(unittest.TestCase):class MaxRetryTimesTest(unittest.TestCase):-invalid_url = "http://www.scrapytest.org/invalid_url"def get_spider_and_middleware(self, settings=None):diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 02b59f448..410eba921 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -243,7 +243,6 @@ class CrawlerRun:class EngineTest(unittest.TestCase):@defer.inlineCallbacksdef test_crawler(self):-for spider in (TestSpider,DictItemsSpider,diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 8e0999348..1491e788e 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -55,7 +55,6 @@ class CustomFieldDataclass:class BaseItemExporterTest(unittest.TestCase):-item_class = TestItemcustom_field_item_class = CustomFieldItem@@ -513,13 +512,11 @@ class XmlItemExporterTest(BaseItemExporterTest):class XmlItemExporterDataclassTest(XmlItemExporterTest):-item_class = TestDataClasscustom_field_item_class = CustomFieldDataclassclass JsonLinesItemExporterTest(BaseItemExporterTest):-_expected_nested = {"name": "Jesus","age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}},@@ -559,13 +556,11 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest):-item_class = TestDataClasscustom_field_item_class = CustomFieldDataclassclass JsonItemExporterTest(JsonLinesItemExporterTest):-_expected_nested = [JsonLinesItemExporterTest._expected_nested]def _get_exporter(self, **kwargs):@@ -627,13 +622,11 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):class JsonItemExporterDataclassTest(JsonItemExporterTest):-item_class = TestDataClasscustom_field_item_class = CustomFieldDataclassclass CustomExporterItemTest(unittest.TestCase):-item_class = TestItemdef setUp(self):@@ -664,7 +657,6 @@ class CustomExporterItemTest(unittest.TestCase):class CustomExporterDataclassTest(CustomExporterItemTest):-item_class = TestDataClassdiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 96f97ca99..eafe1b334 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1068,7 +1068,6 @@ class FeedExportTest(FeedExportTestBase):@defer.inlineCallbacksdef test_export_multiple_item_classes(self):-items = [self.MyItem({"foo": "bar1", "egg": "spam1"}),self.MyItem2({"hello": "world2", "foo": "bar2"}),@@ -1711,7 +1710,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):@defer.inlineCallbacksdef test_gzip_plugin(self):-filename = self._named_tempfile("gzip_file")settings = {@@ -1731,7 +1729,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):@defer.inlineCallbacksdef test_gzip_plugin_compresslevel(self):-filename_to_compressed = {self._named_tempfile("compresslevel_0"): self.get_gzip_compressed(self.expected, compresslevel=0@@ -1839,7 +1836,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):@defer.inlineCallbacksdef test_lzma_plugin(self):-filename = self._named_tempfile("lzma_file")settings = {@@ -1859,7 +1855,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):@defer.inlineCallbacksdef test_lzma_plugin_format(self):-filename_to_compressed = {self._named_tempfile("format_FORMAT_XZ"): lzma.compress(self.expected, format=lzma.FORMAT_XZ@@ -1893,7 +1888,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):@defer.inlineCallbacksdef test_lzma_plugin_check(self):-filename_to_compressed = {self._named_tempfile("check_CHECK_NONE"): lzma.compress(self.expected, check=lzma.CHECK_NONE@@ -1927,7 +1921,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):@defer.inlineCallbacksdef test_lzma_plugin_preset(self):-filename_to_compressed = {self._named_tempfile("preset_PRESET_0"): lzma.compress(self.expected, preset=0@@ -1986,7 +1979,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):@defer.inlineCallbacksdef test_bz2_plugin(self):-filename = self._named_tempfile("bz2_file")settings = {@@ -2006,7 +1998,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):@defer.inlineCallbacksdef test_bz2_plugin_compresslevel(self):-filename_to_compressed = {self._named_tempfile("compresslevel_1"): bz2.compress(self.expected, compresslevel=1@@ -2056,7 +2047,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):@defer.inlineCallbacksdef test_custom_plugin_with_parameter(self):-expected = b"foo\r\n\nbar\r\n\n"filename = self._named_tempfile("newline")@@ -2075,7 +2065,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):@defer.inlineCallbacksdef test_custom_plugin_with_compression(self):-expected = b"foo\r\n\nbar\r\n\n"filename_to_decompressor = {@@ -2555,7 +2544,6 @@ class BatchDeliveriesTest(FeedExportTestBase):]class CustomS3FeedStorage(S3FeedStorage):-stubs = []def open(self, *args, **kwargs):@@ -2828,7 +2816,6 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):class URIParamsTest:-spider_name = "uri_params_spider"deprecated_options = Falsediff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex 5a7413b32..2bd68f846 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -19,7 +19,6 @@ from scrapy.utils.python import to_bytes, to_unicodeclass RequestTest(unittest.TestCase):-request_class = Requestdefault_method = "GET"default_headers = {}@@ -424,7 +423,6 @@ class RequestTest(unittest.TestCase):class FormRequestTest(RequestTest):-request_class = FormRequestdef assertQueryEqual(self, first, second, msg=None):@@ -1447,7 +1445,6 @@ def _qs(req, encoding="utf-8", to_unicode=False):class XmlRpcRequestTest(RequestTest):-request_class = XmlRpcRequestdefault_method = "POST"default_headers = {b"Content-Type": [b"text/xml"]}diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex beb0b712b..dbc9f1fef 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -23,7 +23,6 @@ from tests import get_testdataclass BaseResponseTest(unittest.TestCase):-response_class = Responsedef test_init(self):@@ -349,7 +348,6 @@ class BaseResponseTest(unittest.TestCase):class TextResponseTest(BaseResponseTest):-response_class = TextResponsedef test_replace(self):@@ -835,11 +833,9 @@ class TextResponseTest(BaseResponseTest):class HtmlResponseTest(TextResponseTest):-response_class = HtmlResponsedef test_html_encoding(self):-body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"></head><body>Price: \xa3100</body></html>'@@ -878,7 +874,6 @@ class HtmlResponseTest(TextResponseTest):class XmlResponseTest(TextResponseTest):-response_class = XmlResponsedef test_xml_encoding(self):diff --git a/tests/test_loader.py b/tests/test_loader.pyindex 5f4750ff3..0dc2de172 100644--- a/tests/test_loader.py+++ b/tests/test_loader.py@@ -87,7 +87,6 @@ class BasicItemLoaderTest(unittest.TestCase):class InitializationTestMixin:-item_class = Nonedef test_keep_single_value(self):diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.pyindex 638af825b..3eae87599 100644--- a/tests/test_loader_deprecated.py+++ b/tests/test_loader_deprecated.py@@ -469,7 +469,6 @@ class BasicItemLoaderTest(unittest.TestCase):class InitializationFromDictTest(unittest.TestCase):-item_class = dictdef test_keep_single_value(self):diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex 0971a5a38..56810f2ff 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -16,7 +16,6 @@ from tests.spiders import ItemSpiderclass CustomItem(Item):-name = Field()def __str__(self):diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 8f5d87ebf..cb9464949 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -116,7 +116,6 @@ class FileDownloadCrawlTestCase(TestCase):self.assertTrue((self.tmpmediastore / i["path"]).exists())def _assert_files_download_failure(self, crawler, items, code, logs):-# check that the item does NOT have the "images/files" field populatedself.assertEqual(len(items), 1)self.assertIn(self.media_key, items[0])@@ -205,7 +204,6 @@ else:class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase):-skip = skip_pillowpipeline_class = "scrapy.pipelines.images.ImagesPipeline"diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex cfdac1ab8..6fd8e6308 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -33,7 +33,6 @@ else:class ImagesPipelineTestCase(unittest.TestCase):-skip = skip_pillowdef setUp(self):@@ -325,7 +324,6 @@ class DeprecatedImagesPipeline(ImagesPipeline):class ImagesPipelineTestCaseFieldsMixin:-skip = skip_pillowdef test_item_fields_default(self):@@ -420,7 +418,6 @@ class ImagesPipelineTestCaseFieldsAttrsItem(class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):-skip = skip_pillowimg_cls_attribute_names = [diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 66a492217..d655eb128 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -37,7 +37,6 @@ def _mocked_download_func(request, info):class BaseMediaPipelineTestCase(unittest.TestCase):-pipeline_class = MediaPipelinesettings = None@@ -213,7 +212,6 @@ class MockedMediaPipeline(MediaPipeline):class MediaPipelineTestCase(BaseMediaPipelineTestCase):-pipeline_class = MockedMediaPipelinedef _callback(self, result):diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex 577522c6c..adce6cda7 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -154,7 +154,6 @@ class KeywordArgumentsSpider(MockServerSpider):class CallbackKeywordArgumentsTestCase(TestCase):-maxDiff = Nonedef setUp(self):diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex 5acc412e5..bfb370373 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -43,7 +43,6 @@ class MockDownloader:class MockCrawler(Crawler):def __init__(self, priority_queue_cls, jobdir):-settings = dict(SCHEDULER_DEBUG=False,SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue",@@ -325,7 +324,6 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase):@defer.inlineCallbacksdef test_integration_downloader_aware_priority_queue(self):with MockServer() as mockserver:-url = mockserver.url("/status?n=200", is_secure=False)start_urls = [url] * 6yield self.crawler.crawl(start_urls)diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 2a3b2d529..3e8187096 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -93,7 +93,6 @@ class BaseSettingsTest(unittest.TestCase):with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object(attr, "set") as mock_set:-self.settings.attributes = {"TEST_OPTION": attr}for priority in (0, 10, 20):diff --git a/tests/test_spider.py b/tests/test_spider.pyindex eb8a1f9f0..57c2b79cb 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -26,7 +26,6 @@ from tests import get_testdataclass SpiderTest(unittest.TestCase):-spider_class = Spiderdef setUp(self):@@ -115,12 +114,10 @@ class SpiderTest(unittest.TestCase):class InitSpiderTest(SpiderTest):-spider_class = InitSpiderclass XMLFeedSpiderTest(SpiderTest):-spider_class = XMLFeedSpiderdef test_register_namespace(self):@@ -174,7 +171,6 @@ class XMLFeedSpiderTest(SpiderTest):class CSVFeedSpiderTest(SpiderTest):-spider_class = CSVFeedSpiderdef test_parse_rows(self):@@ -196,7 +192,6 @@ class CSVFeedSpiderTest(SpiderTest):class CrawlSpiderTest(SpiderTest):-test_body = b"""<html><head><title>Page title<title><body><p><a href="item/12.html">Item 12</a></p>@@ -210,7 +205,6 @@ class CrawlSpiderTest(SpiderTest):spider_class = CrawlSpiderdef test_rule_without_link_extractor(self):-response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)@@ -234,7 +228,6 @@ class CrawlSpiderTest(SpiderTest):)def test_process_links(self):-response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)@@ -261,7 +254,6 @@ class CrawlSpiderTest(SpiderTest):)def test_process_links_filter(self):-response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)@@ -290,7 +282,6 @@ class CrawlSpiderTest(SpiderTest):)def test_process_links_generator(self):-response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)@@ -318,7 +309,6 @@ class CrawlSpiderTest(SpiderTest):)def test_process_request(self):-response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)@@ -347,7 +337,6 @@ class CrawlSpiderTest(SpiderTest):)def test_process_request_with_response(self):-response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)@@ -383,7 +372,6 @@ class CrawlSpiderTest(SpiderTest):)def test_process_request_instance_method(self):-response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)@@ -410,7 +398,6 @@ class CrawlSpiderTest(SpiderTest):)def test_process_request_instance_method_with_response(self):-response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)@@ -467,7 +454,6 @@ class CrawlSpiderTest(SpiderTest):class SitemapSpiderTest(SpiderTest):-spider_class = SitemapSpiderBODY = b"SITEMAP"@@ -689,7 +675,6 @@ class DeprecationTest(unittest.TestCase):class NoParseMethodSpiderTest(unittest.TestCase):-spider_class = Spiderdef test_undefined_parse_method(self):diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex da656303d..04025d30d 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -114,13 +114,11 @@ class SpiderLoaderTest(unittest.TestCase):self.assertEqual(crawler.spidercls.name, "spider1")def test_bad_spider_modules_exception(self):-module = "tests.test_spiderloader.test_spiders.doesnotexist"settings = Settings({"SPIDER_MODULES": [module]})self.assertRaises(ImportError, SpiderLoader.from_settings, settings)def test_bad_spider_modules_warning(self):-with warnings.catch_warnings(record=True) as w:module = "tests.test_spiderloader.test_spiders.doesnotexist"settings = Settings(diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 1bc5ccb9a..2f0dda269 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -31,7 +31,6 @@ from scrapy.spiders import Spiderclass TestRefererMiddleware(TestCase):-req_meta = {}resp_headers = {}settings = {}@@ -51,7 +50,6 @@ class TestRefererMiddleware(TestCase):return Response(origin, headers=self.resp_headers)def test(self):-for origin, target, referrer in self.scenarii:response = self.get_response(origin)request = self.get_request(target)@@ -770,7 +768,6 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):class TestRequestMetaSettingFallback(TestCase):-params = [(# When an unknown policy is referenced in Request.meta@@ -824,7 +821,6 @@ class TestRequestMetaSettingFallback(TestCase):]def test(self):-origin = "http://www.scrapy.org"target = "http://www.example.com"@@ -923,7 +919,6 @@ class TestPolicyHeaderPrecedence004(class TestReferrerOnRedirect(TestRefererMiddleware):-settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}scenarii = [(@@ -966,7 +961,6 @@ class TestReferrerOnRedirect(TestRefererMiddleware):self.redirectmw = RedirectMiddleware(settings)def test(self):-for (parent,target,diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex 1586f90c5..4ce7cc9a4 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -73,7 +73,6 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):-chunksize = 100000def queue(self):diff --git a/tests/test_utils_console.py b/tests/test_utils_console.pyindex 18f63b540..dabd6054d 100644--- a/tests/test_utils_console.py+++ b/tests/test_utils_console.py@@ -20,7 +20,6 @@ except ImportError:class UtilsConsoleTestCase(unittest.TestCase):def test_get_shell_embed_func(self):-shell = get_shell_embed_func(["invalid"])self.assertEqual(shell, None)@@ -30,14 +29,12 @@ class UtilsConsoleTestCase(unittest.TestCase):@unittest.skipIf(not bpy, "bpython not available in testenv")def test_get_shell_embed_func2(self):-shell = get_shell_embed_func(["bpython"])self.assertTrue(callable(shell))self.assertEqual(shell.__name__, "_embed_bpython_shell")@unittest.skipIf(not ipy, "IPython not available in testenv")def test_get_shell_embed_func3(self):-# default shell should be 'ipython'shell = get_shell_embed_func()self.assertEqual(shell.__name__, "_embed_ipython_shell")diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.pyindex ed077440c..bab6e9b68 100644--- a/tests/test_utils_iterators.py+++ b/tests/test_utils_iterators.py@@ -7,7 +7,6 @@ from tests import get_testdataclass XmliterTestCase(unittest.TestCase):-xmliter = staticmethod(xmliter)def test_xmliter(self):diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 4760bf92b..6ca272de1 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -462,7 +462,6 @@ class BackwardCompatibilityTestCase(unittest.TestCase):warning to be logged."""class RequestFingerprinter:-cache = WeakKeyDictionary()def fingerprint(self, request):@@ -641,7 +640,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):def test_from_crawler_and_settings(self):class RequestFingerprinter:-# This method is ignored due to the presence of from_crawler@classmethoddef from_settings(cls, settings):diff --git a/tests/test_utils_template.py b/tests/test_utils_template.pyindex c79a1fdce..cbe80e157 100644--- a/tests/test_utils_template.py+++ b/tests/test_utils_template.py@@ -16,7 +16,6 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):rmtree(self.tmp_path)def test_simple_render(self):-context = dict(project_name="proj", name="spi", classname="TheSpider")template = "from ${project_name}.spiders.${name} import ${classname}"rendered = "from proj.spiders.spi import TheSpider"diff --git a/tox.ini b/tox.iniindex 453c28c4c..7385e4468 100644--- a/tox.ini+++ b/tox.ini@@ -4,7 +4,7 @@# and then run "tox" from this directory.[tox]-envlist = security,flake8,black,typing,py+envlist = pre-commit,typing,pyminversion = 1.7.0[testenv]@@ -37,44 +37,34 @@ install_command =[testenv:typing]basepython = python3deps =- lxml-stubs==0.2.0+ lxml-stubs==0.4.0mypy==0.991types-attrs==19.1.0- types-pyOpenSSL==21.0.0- types-setuptools==57.0.0+ types-pyOpenSSL==23.0.0.2+ types-setuptools==65.7.0.3commands =mypy --show-error-codes {posargs: scrapy tests}-[testenv:security]+[testenv:pre-commit]basepython = python3deps =- bandit==1.7.4+ pre-commitcommands =- bandit -r -c .bandit.yml {posargs:scrapy}--[testenv:flake8]-basepython = python3-deps =- {[testenv]deps}- # Twisted[http2] is required to import some files- Twisted[http2]>=17.9.0- flake8==6.0.0-commands =- flake8 {posargs:docs scrapy tests}+ pre-commit run {posargs:--all-files}[testenv:pylint]basepython = python3deps ={[testenv:extra-deps]deps}- pylint==2.15.6+ pylint==2.16.0commands =pylint conftest.py docs extras scrapy setup.py tests[testenv:twinecheck]basepython = python3deps =- twine==4.0.1- build==0.9.0+ twine==4.0.2+ build==0.10.0commands =python -m build --sdisttwine check dist/*@@ -195,9 +185,3 @@ deps = {[docs]deps}setenv = {[docs]setenv}commands =sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck--[testenv:black]-deps =- black==22.12.0-commands =- black {posargs:--check .}