Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Upgrade CI tools

Adrián Chavesb07d3f85a389
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f5fc1285f..6840bfe23 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,14 +5,14 @@ repos:
- id: bandit
args: [-r, -c, .bandit.yml]
- repo: https://github.com/PyCQA/flake8
- rev: 6.0.0
+ rev: 5.0.4 # 6.0.0 drops Python 3.7 support
hooks:
- id: flake8
- repo: https://github.com/psf/black.git
- rev: 22.12.0
+ rev: 23.1.0
hooks:
- id: black
- repo: https://github.com/pycqa/isort
- rev: 5.12.0
+ rev: 5.11.5 # 5.12 drops Python 3.7 support
hooks:
- id: isort
diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index efb4c202e..1f270837c 100644
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -18,7 +18,6 @@ from pathlib import Path
def main():
-
# Used for remembering the file (and its contents)
# so we don't have to open the same file again.
_filename = None
@@ -50,7 +49,6 @@ def main():
else:
# If this is a new file
if newfilename != _filename:
-
# Update the previous file
if _filename:
Path(_filename).write_text(_contents, encoding="utf-8")
diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index acad71e07..119dfdabb 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -13,7 +13,6 @@ from scrapy.spiders import Spider
class QPSSpider(Spider):
-
name = "qps"
benchurl = "http://localhost:8880/"
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index de68c43a5..9baee3a48 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -14,7 +14,6 @@ from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
class ScrapyCommand:
-
requires_project = False
crawler_process: Optional[CrawlerProcess] = None
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 911e5afe6..e1ccdc451 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -9,7 +9,6 @@ from scrapy.linkextractors import LinkExtractor
class Command(ScrapyCommand):
-
default_settings = {
"LOG_LEVEL": "INFO",
"LOGSTATS_INTERVAL": 1,
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index df8006f36..2f0f1c7b9 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -3,7 +3,6 @@ from scrapy.exceptions import UsageError
class Command(BaseRunSpiderCommand):
-
requires_project = True
def syntax(self):
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index ca591011c..03a8ed5c7 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -6,7 +6,6 @@ from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
-
requires_project = True
default_settings = {"LOG_ENABLED": False}
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index a9076c5b1..1359e445f 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -10,7 +10,6 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request
class Command(ScrapyCommand):
-
requires_project = False
def syntax(self):
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 90dd0874e..c1565a138 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -32,7 +32,6 @@ def extract_domain(url):
class Command(ScrapyCommand):
-
requires_project = False
default_settings = {"LOG_ENABLED": False}
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index 2a81bd182..2f5032360 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -2,7 +2,6 @@ from scrapy.commands import ScrapyCommand
class Command(ScrapyCommand):
-
requires_project = True
default_settings = {"LOG_ENABLED": False}
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 8a75f9270..58ed89a81 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -24,7 +24,6 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
class Command(BaseRunSpiderCommand):
-
requires_project = False
default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index c43298ac7..318187204 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -5,7 +5,6 @@ from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
-
requires_project = False
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 05c76d1eb..63c23d04c 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -13,7 +13,6 @@ from scrapy.utils.url import guess_scheme
class Command(ScrapyCommand):
-
requires_project = False
default_settings = {
"KEEP_ALIVE": True,
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 88bd5bb33..fde609c6f 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -28,7 +28,6 @@ def _make_writable(path):
class Command(ScrapyCommand):
-
requires_project = False
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index faa0e242c..47582866b 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -4,7 +4,6 @@ from scrapy.utils.versions import scrapy_components_versions
class Command(ScrapyCommand):
-
default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
def syntax(self):
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 7e0b62bb0..7d27a5193 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -69,7 +69,6 @@ def _get_concurrency_delay(concurrency, spider, settings):
class Downloader:
-
DOWNLOAD_SLOT = "download_slot"
def __init__(self, crawler):
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 8de5459e9..c3704de3d 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -292,7 +292,6 @@ class ScrapyProxyAgent(Agent):
class ScrapyAgent:
-
_Agent = Agent
_ProxyAgent = ScrapyProxyAgent
_TunnelingAgent = TunnelingAgent
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 5a94e66a6..56df48b2e 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -17,7 +17,6 @@ from scrapy.utils.defer import deferred_from_coro, mustbe_deferred
class DownloaderMiddlewareManager(MiddlewareManager):
-
component_name = "downloader middleware"
@classmethod
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 4558402b2..3d103652b 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -40,7 +40,6 @@ def _parse(url):
class ScrapyHTTPPageGetter(HTTPClient):
-
delimiter = b"\n"
def connectionMade(self):
@@ -103,7 +102,6 @@ class ScrapyHTTPPageGetter(HTTPClient):
# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
# non-overridden code into this class.
class ScrapyHTTPClientFactory(ClientFactory):
-
protocol = ScrapyHTTPPageGetter
waiting = 1
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index ba9c37e38..971cf52e4 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -46,7 +46,6 @@ def _isiterable(o) -> bool:
class SpiderMiddlewareManager(MiddlewareManager):
-
component_name = "spider middleware"
def __init__(self, *middlewares):
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 137ed5b18..04ae719de 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -30,7 +30,6 @@ class AjaxCrawlMiddleware:
return cls(crawler.settings)
def process_response(self, request, response, spider):
-
if not isinstance(response, HtmlResponse) or response.status != 200:
return response
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 74c55f6e2..b9316c43a 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -27,7 +27,6 @@ HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddlew
class HttpCacheMiddleware:
-
DOWNLOAD_EXCEPTIONS = (
defer.TimeoutError,
TimeoutError,
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 7b5f4824a..ead426951 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -53,7 +53,6 @@ class HttpCompressionMiddleware:
request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
def process_response(self, request, response, spider):
-
if request.method == "HEAD":
return response
if isinstance(response, Response):
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index f442a3012..65f1d2224 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -26,7 +26,6 @@ def _build_redirect_request(source_request, *, url, **kwargs):
class BaseRedirectMiddleware:
-
enabled_setting = "REDIRECT_ENABLED"
def __init__(self, settings):
@@ -115,7 +114,6 @@ class RedirectMiddleware(BaseRedirectMiddleware):
class MetaRefreshMiddleware(BaseRedirectMiddleware):
-
enabled_setting = "METAREFRESH_ENABLED"
def __init__(self, settings):
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 11a30911c..081642a4b 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -122,7 +122,6 @@ def get_retry_request(
class RetryMiddleware:
-
# IOError is raised by the HttpCompression middleware when trying to
# decompress an empty response
EXCEPTIONS_TO_RETRY = (
diff --git a/scrapy/extension.py b/scrapy/extension.py
index 89f6a694d..4e365cfa1 100644
--- a/scrapy/extension.py
+++ b/scrapy/extension.py
@@ -8,7 +8,6 @@ from scrapy.utils.conf import build_component_list
class ExtensionManager(MiddlewareManager):
-
component_name = "extension"
@classmethod
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 2d120a6ed..dfe843974 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -41,7 +41,6 @@ class DummyPolicy:
class RFC2616Policy:
-
MAXAGE = 3600 * 24 * 365 # one year
def __init__(self, settings):
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 35e2808c2..510c903db 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -15,7 +15,6 @@ from scrapy.utils.deprecate import create_deprecated_class
class JsonRequest(Request):
-
attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None:
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index e45d95602..73bb811de 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -29,7 +29,6 @@ _NONE = object()
class TextResponse(Response):
-
_DEFAULT_ENCODING = "ascii"
_cached_decoded_json = _NONE
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 66cf58d89..dd8dcdf7c 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -194,7 +194,6 @@ class LxmlLinkExtractor:
return True
def matches(self, url):
-
if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):
return False
if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 536341fc6..df574a0a1 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -10,7 +10,6 @@ from scrapy.utils.defer import deferred_f_from_coro_f
class ItemPipelineManager(MiddlewareManager):
-
component_name = "item pipeline"
@classmethod
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 3430603ab..0be28933e 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -187,7 +187,6 @@ class S3FilesStore:
class GCSFilesStore:
-
GCS_PROJECT_ID = None
CACHE_CONTROL = "max-age=172800"
@@ -253,7 +252,6 @@ class GCSFilesStore:
class FTPFilesStore:
-
FTP_USERNAME = None
FTP_PASSWORD = None
USE_ACTIVE_MODE = None
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 31f51c004..401029439 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -23,7 +23,6 @@ def _DUMMY_CALLBACK(response):
class MediaPipeline:
-
LOG_FAILED_RESULTS = True
class SpiderInfo:
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 6af8915c2..f01e9096c 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -12,7 +12,6 @@ from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
class ResponseTypes:
-
CLASSES = {
"text/html": "scrapy.http.HtmlResponse",
"application/atom+xml": "scrapy.http.XmlResponse",
diff --git a/scrapy/shell.py b/scrapy/shell.py
index ae6e641fd..bb3b1461c 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -25,7 +25,6 @@ from scrapy.utils.response import open_in_browser
class Shell:
-
relevant_classes = (Crawler, Spider, Request, Response, Settings)
def __init__(self, crawler, update_vars=None, code=None):
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index d86f55a40..fd91e658b 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -34,7 +34,6 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default"
class ReferrerPolicy:
-
NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES
name: str
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 05c425948..31e845716 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -60,7 +60,6 @@ class Rule:
class CrawlSpider(Spider):
-
rules: Sequence[Rule] = ()
def __init__(self, *a, **kw):
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 5ec0504a8..6afadc577 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -89,7 +89,7 @@ class XMLFeedSpider(Spider):
yield node
def _register_namespaces(self, selector):
- for (prefix, uri) in self.namespaces:
+ for prefix, uri in self.namespaces:
selector.register_namespace(prefix, uri)
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index c3cca9699..aaf75a519 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -10,7 +10,6 @@ logger = logging.getLogger(__name__)
class SitemapSpider(Spider):
-
sitemap_urls = ()
sitemap_rules = [("", "parse")]
sitemap_follow = [""]
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 1089ba7b8..38884a9f0 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -6,7 +6,6 @@ from twisted.web.server import Site
class Root(Resource):
-
isLeaf = True
def getChild(self, name, request):
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 10497e9ed..fa57a4f26 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -11,7 +11,6 @@ from collections.abc import Mapping
class CaselessDict(dict):
-
__slots__ = ()
def __init__(self, seq=None):
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index 61a4347ea..f4d6e0451 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -54,7 +54,6 @@ def create_deprecated_class(
"""
class DeprecatedClass(new_class.__class__):
-
deprecated_class = None
warned_on_subclass = False
diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py
index 358f41679..414658944 100644
--- a/scrapy/utils/serialize.py
+++ b/scrapy/utils/serialize.py
@@ -9,7 +9,6 @@ from scrapy.http import Request, Response
class ScrapyJSONEncoder(json.JSONEncoder):
-
DATE_FORMAT = "%Y-%m-%d"
TIME_FORMAT = "%H:%M:%S"
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index ecb2e31bf..5f9bdef37 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -5,7 +5,6 @@ from twisted.internet import defer, protocol
class ProcessTest:
-
command = None
prefix = [sys.executable, "-m", "scrapy.cmdline"]
cwd = os.getcwd() # trial chdirs to temp dir
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 0a27ccd6d..833aa3e20 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -144,7 +144,6 @@ def strip_url(
origin_only=False,
strip_fragment=True,
):
-
"""Strip URL string from some of its components:
- ``strip_credentials`` removes "user:password@"
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 7991da9dc..d541f3f7c 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -95,7 +95,6 @@ class BrokenDownloadResource(resource.Resource):
class LeafResource(resource.Resource):
-
isLeaf = True
def deferRequest(self, request, delay, f, *a, **kw):
diff --git a/tests/requirements.txt b/tests/requirements.txt
index d9373dfa8..618949795 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -2,7 +2,7 @@
attrs
pyftpdlib
pytest
-pytest-cov==3.0.0
+pytest-cov==4.0.0
pytest-xdist
sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
testfixtures
diff --git a/tests/spiders.py b/tests/spiders.py
index 2036e8cf3..6ff48f471 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -25,7 +25,6 @@ class MockServerSpider(Spider):
class MetaSpider(MockServerSpider):
-
name = "meta"
def __init__(self, *args, **kwargs):
@@ -37,7 +36,6 @@ class MetaSpider(MockServerSpider):
class FollowAllSpider(MetaSpider):
-
name = "follow"
link_extractor = LinkExtractor()
@@ -59,7 +57,6 @@ class FollowAllSpider(MetaSpider):
class DelaySpider(MetaSpider):
-
name = "delay"
def __init__(self, n=1, b=0, *args, **kwargs):
@@ -81,7 +78,6 @@ class DelaySpider(MetaSpider):
class SimpleSpider(MetaSpider):
-
name = "simple"
def __init__(self, url="http://localhost:8998", *args, **kwargs):
@@ -93,7 +89,6 @@ class SimpleSpider(MetaSpider):
class AsyncDefSpider(SimpleSpider):
-
name = "asyncdef"
async def parse(self, response):
@@ -102,7 +97,6 @@ class AsyncDefSpider(SimpleSpider):
class AsyncDefAsyncioSpider(SimpleSpider):
-
name = "asyncdef_asyncio"
async def parse(self, response):
@@ -112,7 +106,6 @@ class AsyncDefAsyncioSpider(SimpleSpider):
class AsyncDefAsyncioReturnSpider(SimpleSpider):
-
name = "asyncdef_asyncio_return"
async def parse(self, response):
@@ -123,7 +116,6 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
-
name = "asyncdef_asyncio_return_single_element"
async def parse(self, response):
@@ -134,7 +126,6 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider):
class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
-
name = "asyncdef_asyncio_reqs_return"
async def parse(self, response):
@@ -191,7 +182,6 @@ class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
class AsyncDefAsyncioGenSpider(SimpleSpider):
-
name = "asyncdef_asyncio_gen"
async def parse(self, response):
@@ -201,7 +191,6 @@ class AsyncDefAsyncioGenSpider(SimpleSpider):
class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
-
name = "asyncdef_asyncio_gen_loop"
async def parse(self, response):
@@ -212,7 +201,6 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
-
name = "asyncdef_asyncio_gen_complex"
initial_reqs = 4
following_reqs = 3
@@ -246,7 +234,6 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
class ItemSpider(FollowAllSpider):
-
name = "item"
def parse(self, response):
@@ -261,7 +248,6 @@ class DefaultError(Exception):
class ErrorSpider(FollowAllSpider):
-
name = "error"
exception_cls = DefaultError
@@ -275,7 +261,6 @@ class ErrorSpider(FollowAllSpider):
class BrokenStartRequestsSpider(FollowAllSpider):
-
fail_before_yield = False
fail_yielding = False
@@ -305,7 +290,6 @@ class BrokenStartRequestsSpider(FollowAllSpider):
class SingleRequestSpider(MetaSpider):
-
seed = None
callback_func = None
errback_func = None
@@ -451,7 +435,6 @@ class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParse
class BytesReceivedCallbackSpider(MetaSpider):
-
full_response_length = 2**18
@classmethod
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 1f299587f..129ef0121 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -2,7 +2,6 @@ from tests.test_commands import CommandTest
class CheckCommandTest(CommandTest):
-
command = "check"
def setUp(self):
diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py
index 124c968c2..d2027d1c2 100644
--- a/tests/test_command_fetch.py
+++ b/tests/test_command_fetch.py
@@ -6,7 +6,6 @@ from scrapy.utils.testsite import SiteTest
class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
-
command = "fetch"
@defer.inlineCallbacks
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 8ce82db86..6589381f3 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -9,7 +9,6 @@ from tests import NON_EXISTING_RESOLVABLE, tests_datadir
class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
-
command = "shell"
@defer.inlineCallbacks
diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index 3bf6019b5..a52d0d13c 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -8,7 +8,6 @@ from scrapy.utils.testproc import ProcessTest
class VersionTest(ProcessTest, unittest.TestCase):
-
command = "version"
@defer.inlineCallbacks
diff --git a/tests/test_commands.py b/tests/test_commands.py
index d7ee75559..00ddcdd3e 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -223,7 +223,6 @@ def get_permissions_dict(
class StartprojectTemplatesTest(ProjectTest):
-
maxDiff = None
def setUp(self):
@@ -604,7 +603,6 @@ class MiscCommandsTest(CommandTest):
class RunSpiderCommandTest(CommandTest):
-
spider_filename = "myspider.py"
debug_log_spider = """
@@ -873,7 +871,6 @@ class MySpider(scrapy.Spider):
@skipIf(platform.system() != "Windows", "Windows required for .pyw files")
class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
-
spider_filename = "myspider.pyw"
def setUp(self):
diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py
index 8090d50b9..31fa1430d 100644
--- a/tests/test_downloader_handlers_http2.py
+++ b/tests/test_downloader_handlers_http2.py
@@ -25,7 +25,6 @@ from tests.test_downloader_handlers import (
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2TestCase(Https11TestCase):
-
scheme = "https"
HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 2be32e37b..060cfe08b 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -16,7 +16,6 @@ from scrapy.utils.test import get_crawler, get_from_asyncio_queue
class ManagerTestCase(TestCase):
-
settings_dict = None
def setUp(self):
diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py
index 412c20a78..95739414e 100644
--- a/tests/test_downloadermiddleware_decompression.py
+++ b/tests/test_downloadermiddleware_decompression.py
@@ -17,7 +17,6 @@ def _test_data(formats):
class DecompressionMiddlewareTest(TestCase):
-
test_formats = ["tar", "xml.bz2", "xml.gz", "zip"]
uncompressed_body, test_responses = _test_data(test_formats)
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index a355a9b5b..f80eff3e6 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -14,7 +14,6 @@ from scrapy.utils.test import get_crawler
class _BaseTest(unittest.TestCase):
-
storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
@@ -146,12 +145,10 @@ class DefaultStorageTest(_BaseTest):
class DbmStorageTest(DefaultStorageTest):
-
storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
-
dbm_module = "tests.mocks.dummydbm"
def _get_settings(self, **new_settings):
@@ -165,7 +162,6 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
class FilesystemStorageTest(DefaultStorageTest):
-
storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
@@ -176,7 +172,6 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
class DummyPolicyTest(_BaseTest):
-
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
def test_middleware(self):
@@ -270,7 +265,6 @@ class DummyPolicyTest(_BaseTest):
class RFC2616PolicyTest(DefaultStorageTest):
-
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
def _process_requestresponse(self, mw, request, response):
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index ca125ba36..9ec51f75b 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -13,7 +13,6 @@ spider = Spider("foo")
class TestHttpProxyMiddleware(TestCase):
-
failureException = AssertionError
def setUp(self):
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 02854c2a7..63bd61848 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -127,7 +127,6 @@ class RetryTest(unittest.TestCase):
class MaxRetryTimesTest(unittest.TestCase):
-
invalid_url = "http://www.scrapytest.org/invalid_url"
def get_spider_and_middleware(self, settings=None):
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 02b59f448..410eba921 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -243,7 +243,6 @@ class CrawlerRun:
class EngineTest(unittest.TestCase):
@defer.inlineCallbacks
def test_crawler(self):
-
for spider in (
TestSpider,
DictItemsSpider,
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 8e0999348..1491e788e 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -55,7 +55,6 @@ class CustomFieldDataclass:
class BaseItemExporterTest(unittest.TestCase):
-
item_class = TestItem
custom_field_item_class = CustomFieldItem
@@ -513,13 +512,11 @@ class XmlItemExporterTest(BaseItemExporterTest):
class XmlItemExporterDataclassTest(XmlItemExporterTest):
-
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class JsonLinesItemExporterTest(BaseItemExporterTest):
-
_expected_nested = {
"name": "Jesus",
"age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}},
@@ -559,13 +556,11 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest):
-
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class JsonItemExporterTest(JsonLinesItemExporterTest):
-
_expected_nested = [JsonLinesItemExporterTest._expected_nested]
def _get_exporter(self, **kwargs):
@@ -627,13 +622,11 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
class JsonItemExporterDataclassTest(JsonItemExporterTest):
-
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass
class CustomExporterItemTest(unittest.TestCase):
-
item_class = TestItem
def setUp(self):
@@ -664,7 +657,6 @@ class CustomExporterItemTest(unittest.TestCase):
class CustomExporterDataclassTest(CustomExporterItemTest):
-
item_class = TestDataClass
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 96f97ca99..eafe1b334 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1068,7 +1068,6 @@ class FeedExportTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_export_multiple_item_classes(self):
-
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
self.MyItem2({"hello": "world2", "foo": "bar2"}),
@@ -1711,7 +1710,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_gzip_plugin(self):
-
filename = self._named_tempfile("gzip_file")
settings = {
@@ -1731,7 +1729,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_gzip_plugin_compresslevel(self):
-
filename_to_compressed = {
self._named_tempfile("compresslevel_0"): self.get_gzip_compressed(
self.expected, compresslevel=0
@@ -1839,7 +1836,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_lzma_plugin(self):
-
filename = self._named_tempfile("lzma_file")
settings = {
@@ -1859,7 +1855,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_lzma_plugin_format(self):
-
filename_to_compressed = {
self._named_tempfile("format_FORMAT_XZ"): lzma.compress(
self.expected, format=lzma.FORMAT_XZ
@@ -1893,7 +1888,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_lzma_plugin_check(self):
-
filename_to_compressed = {
self._named_tempfile("check_CHECK_NONE"): lzma.compress(
self.expected, check=lzma.CHECK_NONE
@@ -1927,7 +1921,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_lzma_plugin_preset(self):
-
filename_to_compressed = {
self._named_tempfile("preset_PRESET_0"): lzma.compress(
self.expected, preset=0
@@ -1986,7 +1979,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_bz2_plugin(self):
-
filename = self._named_tempfile("bz2_file")
settings = {
@@ -2006,7 +1998,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_bz2_plugin_compresslevel(self):
-
filename_to_compressed = {
self._named_tempfile("compresslevel_1"): bz2.compress(
self.expected, compresslevel=1
@@ -2056,7 +2047,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_custom_plugin_with_parameter(self):
-
expected = b"foo\r\n\nbar\r\n\n"
filename = self._named_tempfile("newline")
@@ -2075,7 +2065,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_custom_plugin_with_compression(self):
-
expected = b"foo\r\n\nbar\r\n\n"
filename_to_decompressor = {
@@ -2555,7 +2544,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
]
class CustomS3FeedStorage(S3FeedStorage):
-
stubs = []
def open(self, *args, **kwargs):
@@ -2828,7 +2816,6 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
class URIParamsTest:
-
spider_name = "uri_params_spider"
deprecated_options = False
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 5a7413b32..2bd68f846 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -19,7 +19,6 @@ from scrapy.utils.python import to_bytes, to_unicode
class RequestTest(unittest.TestCase):
-
request_class = Request
default_method = "GET"
default_headers = {}
@@ -424,7 +423,6 @@ class RequestTest(unittest.TestCase):
class FormRequestTest(RequestTest):
-
request_class = FormRequest
def assertQueryEqual(self, first, second, msg=None):
@@ -1447,7 +1445,6 @@ def _qs(req, encoding="utf-8", to_unicode=False):
class XmlRpcRequestTest(RequestTest):
-
request_class = XmlRpcRequest
default_method = "POST"
default_headers = {b"Content-Type": [b"text/xml"]}
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index beb0b712b..dbc9f1fef 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -23,7 +23,6 @@ from tests import get_testdata
class BaseResponseTest(unittest.TestCase):
-
response_class = Response
def test_init(self):
@@ -349,7 +348,6 @@ class BaseResponseTest(unittest.TestCase):
class TextResponseTest(BaseResponseTest):
-
response_class = TextResponse
def test_replace(self):
@@ -835,11 +833,9 @@ class TextResponseTest(BaseResponseTest):
class HtmlResponseTest(TextResponseTest):
-
response_class = HtmlResponse
def test_html_encoding(self):
-
body = b"""<html><head><title>Some page</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head><body>Price: \xa3100</body></html>'
@@ -878,7 +874,6 @@ class HtmlResponseTest(TextResponseTest):
class XmlResponseTest(TextResponseTest):
-
response_class = XmlResponse
def test_xml_encoding(self):
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 5f4750ff3..0dc2de172 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -87,7 +87,6 @@ class BasicItemLoaderTest(unittest.TestCase):
class InitializationTestMixin:
-
item_class = None
def test_keep_single_value(self):
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index 638af825b..3eae87599 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -469,7 +469,6 @@ class BasicItemLoaderTest(unittest.TestCase):
class InitializationFromDictTest(unittest.TestCase):
-
item_class = dict
def test_keep_single_value(self):
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 0971a5a38..56810f2ff 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -16,7 +16,6 @@ from tests.spiders import ItemSpider
class CustomItem(Item):
-
name = Field()
def __str__(self):
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 8f5d87ebf..cb9464949 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -116,7 +116,6 @@ class FileDownloadCrawlTestCase(TestCase):
self.assertTrue((self.tmpmediastore / i["path"]).exists())
def _assert_files_download_failure(self, crawler, items, code, logs):
-
# check that the item does NOT have the "images/files" field populated
self.assertEqual(len(items), 1)
self.assertIn(self.media_key, items[0])
@@ -205,7 +204,6 @@ else:
class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase):
-
skip = skip_pillow
pipeline_class = "scrapy.pipelines.images.ImagesPipeline"
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index cfdac1ab8..6fd8e6308 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -33,7 +33,6 @@ else:
class ImagesPipelineTestCase(unittest.TestCase):
-
skip = skip_pillow
def setUp(self):
@@ -325,7 +324,6 @@ class DeprecatedImagesPipeline(ImagesPipeline):
class ImagesPipelineTestCaseFieldsMixin:
-
skip = skip_pillow
def test_item_fields_default(self):
@@ -420,7 +418,6 @@ class ImagesPipelineTestCaseFieldsAttrsItem(
class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
-
skip = skip_pillow
img_cls_attribute_names = [
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 66a492217..d655eb128 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -37,7 +37,6 @@ def _mocked_download_func(request, info):
class BaseMediaPipelineTestCase(unittest.TestCase):
-
pipeline_class = MediaPipeline
settings = None
@@ -213,7 +212,6 @@ class MockedMediaPipeline(MediaPipeline):
class MediaPipelineTestCase(BaseMediaPipelineTestCase):
-
pipeline_class = MockedMediaPipeline
def _callback(self, result):
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 577522c6c..adce6cda7 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -154,7 +154,6 @@ class KeywordArgumentsSpider(MockServerSpider):
class CallbackKeywordArgumentsTestCase(TestCase):
-
maxDiff = None
def setUp(self):
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 5acc412e5..bfb370373 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -43,7 +43,6 @@ class MockDownloader:
class MockCrawler(Crawler):
def __init__(self, priority_queue_cls, jobdir):
-
settings = dict(
SCHEDULER_DEBUG=False,
SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue",
@@ -325,7 +324,6 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase):
@defer.inlineCallbacks
def test_integration_downloader_aware_priority_queue(self):
with MockServer() as mockserver:
-
url = mockserver.url("/status?n=200", is_secure=False)
start_urls = [url] * 6
yield self.crawler.crawl(start_urls)
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 2a3b2d529..3e8187096 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -93,7 +93,6 @@ class BaseSettingsTest(unittest.TestCase):
with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object(
attr, "set"
) as mock_set:
-
self.settings.attributes = {"TEST_OPTION": attr}
for priority in (0, 10, 20):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index eb8a1f9f0..57c2b79cb 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -26,7 +26,6 @@ from tests import get_testdata
class SpiderTest(unittest.TestCase):
-
spider_class = Spider
def setUp(self):
@@ -115,12 +114,10 @@ class SpiderTest(unittest.TestCase):
class InitSpiderTest(SpiderTest):
-
spider_class = InitSpider
class XMLFeedSpiderTest(SpiderTest):
-
spider_class = XMLFeedSpider
def test_register_namespace(self):
@@ -174,7 +171,6 @@ class XMLFeedSpiderTest(SpiderTest):
class CSVFeedSpiderTest(SpiderTest):
-
spider_class = CSVFeedSpider
def test_parse_rows(self):
@@ -196,7 +192,6 @@ class CSVFeedSpiderTest(SpiderTest):
class CrawlSpiderTest(SpiderTest):
-
test_body = b"""<html><head><title>Page title<title>
<body>
<p><a href="item/12.html">Item 12</a></p>
@@ -210,7 +205,6 @@ class CrawlSpiderTest(SpiderTest):
spider_class = CrawlSpider
def test_rule_without_link_extractor(self):
-
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
@@ -234,7 +228,6 @@ class CrawlSpiderTest(SpiderTest):
)
def test_process_links(self):
-
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
@@ -261,7 +254,6 @@ class CrawlSpiderTest(SpiderTest):
)
def test_process_links_filter(self):
-
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
@@ -290,7 +282,6 @@ class CrawlSpiderTest(SpiderTest):
)
def test_process_links_generator(self):
-
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
@@ -318,7 +309,6 @@ class CrawlSpiderTest(SpiderTest):
)
def test_process_request(self):
-
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
@@ -347,7 +337,6 @@ class CrawlSpiderTest(SpiderTest):
)
def test_process_request_with_response(self):
-
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
@@ -383,7 +372,6 @@ class CrawlSpiderTest(SpiderTest):
)
def test_process_request_instance_method(self):
-
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
@@ -410,7 +398,6 @@ class CrawlSpiderTest(SpiderTest):
)
def test_process_request_instance_method_with_response(self):
-
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
@@ -467,7 +454,6 @@ class CrawlSpiderTest(SpiderTest):
class SitemapSpiderTest(SpiderTest):
-
spider_class = SitemapSpider
BODY = b"SITEMAP"
@@ -689,7 +675,6 @@ class DeprecationTest(unittest.TestCase):
class NoParseMethodSpiderTest(unittest.TestCase):
-
spider_class = Spider
def test_undefined_parse_method(self):
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index da656303d..04025d30d 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -114,13 +114,11 @@ class SpiderLoaderTest(unittest.TestCase):
self.assertEqual(crawler.spidercls.name, "spider1")
def test_bad_spider_modules_exception(self):
-
module = "tests.test_spiderloader.test_spiders.doesnotexist"
settings = Settings({"SPIDER_MODULES": [module]})
self.assertRaises(ImportError, SpiderLoader.from_settings, settings)
def test_bad_spider_modules_warning(self):
-
with warnings.catch_warnings(record=True) as w:
module = "tests.test_spiderloader.test_spiders.doesnotexist"
settings = Settings(
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 1bc5ccb9a..2f0dda269 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -31,7 +31,6 @@ from scrapy.spiders import Spider
class TestRefererMiddleware(TestCase):
-
req_meta = {}
resp_headers = {}
settings = {}
@@ -51,7 +50,6 @@ class TestRefererMiddleware(TestCase):
return Response(origin, headers=self.resp_headers)
def test(self):
-
for origin, target, referrer in self.scenarii:
response = self.get_response(origin)
request = self.get_request(target)
@@ -770,7 +768,6 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):
class TestRequestMetaSettingFallback(TestCase):
-
params = [
(
# When an unknown policy is referenced in Request.meta
@@ -824,7 +821,6 @@ class TestRequestMetaSettingFallback(TestCase):
]
def test(self):
-
origin = "http://www.scrapy.org"
target = "http://www.example.com"
@@ -923,7 +919,6 @@ class TestPolicyHeaderPrecedence004(
class TestReferrerOnRedirect(TestRefererMiddleware):
-
settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}
scenarii = [
(
@@ -966,7 +961,6 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
self.redirectmw = RedirectMiddleware(settings)
def test(self):
-
for (
parent,
target,
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 1586f90c5..4ce7cc9a4 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -73,7 +73,6 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
-
chunksize = 100000
def queue(self):
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index 18f63b540..dabd6054d 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -20,7 +20,6 @@ except ImportError:
class UtilsConsoleTestCase(unittest.TestCase):
def test_get_shell_embed_func(self):
-
shell = get_shell_embed_func(["invalid"])
self.assertEqual(shell, None)
@@ -30,14 +29,12 @@ class UtilsConsoleTestCase(unittest.TestCase):
@unittest.skipIf(not bpy, "bpython not available in testenv")
def test_get_shell_embed_func2(self):
-
shell = get_shell_embed_func(["bpython"])
self.assertTrue(callable(shell))
self.assertEqual(shell.__name__, "_embed_bpython_shell")
@unittest.skipIf(not ipy, "IPython not available in testenv")
def test_get_shell_embed_func3(self):
-
# default shell should be 'ipython'
shell = get_shell_embed_func()
self.assertEqual(shell.__name__, "_embed_ipython_shell")
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index ed077440c..bab6e9b68 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -7,7 +7,6 @@ from tests import get_testdata
class XmliterTestCase(unittest.TestCase):
-
xmliter = staticmethod(xmliter)
def test_xmliter(self):
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 4760bf92b..6ca272de1 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -462,7 +462,6 @@ class BackwardCompatibilityTestCase(unittest.TestCase):
warning to be logged."""
class RequestFingerprinter:
-
cache = WeakKeyDictionary()
def fingerprint(self, request):
@@ -641,7 +640,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
def test_from_crawler_and_settings(self):
class RequestFingerprinter:
-
# This method is ignored due to the presence of from_crawler
@classmethod
def from_settings(cls, settings):
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index c79a1fdce..cbe80e157 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -16,7 +16,6 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
rmtree(self.tmp_path)
def test_simple_render(self):
-
context = dict(project_name="proj", name="spi", classname="TheSpider")
template = "from ${project_name}.spiders.${name} import ${classname}"
rendered = "from proj.spiders.spi import TheSpider"
diff --git a/tox.ini b/tox.ini
index 453c28c4c..7385e4468 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,7 @@
# and then run "tox" from this directory.
[tox]
-envlist = security,flake8,black,typing,py
+envlist = pre-commit,typing,py
minversion = 1.7.0
[testenv]
@@ -37,44 +37,34 @@ install_command =
[testenv:typing]
basepython = python3
deps =
- lxml-stubs==0.2.0
+ lxml-stubs==0.4.0
mypy==0.991
types-attrs==19.1.0
- types-pyOpenSSL==21.0.0
- types-setuptools==57.0.0
+ types-pyOpenSSL==23.0.0.2
+ types-setuptools==65.7.0.3
commands =
mypy --show-error-codes {posargs: scrapy tests}
-[testenv:security]
+[testenv:pre-commit]
basepython = python3
deps =
- bandit==1.7.4
+ pre-commit
commands =
- bandit -r -c .bandit.yml {posargs:scrapy}
-
-[testenv:flake8]
-basepython = python3
-deps =
- {[testenv]deps}
- # Twisted[http2] is required to import some files
- Twisted[http2]>=17.9.0
- flake8==6.0.0
-commands =
- flake8 {posargs:docs scrapy tests}
+ pre-commit run {posargs:--all-files}
[testenv:pylint]
basepython = python3
deps =
{[testenv:extra-deps]deps}
- pylint==2.15.6
+ pylint==2.16.0
commands =
pylint conftest.py docs extras scrapy setup.py tests
[testenv:twinecheck]
basepython = python3
deps =
- twine==4.0.1
- build==0.9.0
+ twine==4.0.2
+ build==0.10.0
commands =
python -m build --sdist
twine check dist/*
@@ -195,9 +185,3 @@ deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
-
-[testenv:black]
-deps =
- black==22.12.0
-commands =
- black {posargs:--check .}