Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Enable SIM Ruff rules.

Andrey Rakhmatullinc87354cd46af
diff --git a/pyproject.toml b/pyproject.toml
index 08b4b09b2..a75f3b6db 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -254,6 +254,8 @@ extend-select = [
"RUF",
# flake8-bandit
"S",
+ # flake8-simplify
+ "SIM",
# flake8-slots
"SLOT",
# flake8-debugger
@@ -344,6 +346,12 @@ ignore = [
"S321",
# Argument default set to insecure SSL protocol
"S503",
+ # Use capitalized environment variable
+ "SIM112",
+ # Use a context manager for opening files
+ "SIM115",
+ # Yoda condition detected
+ "SIM300",
]
[tool.ruff.lint.per-file-ignores]
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 9a24871de..48f462c65 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -90,12 +90,10 @@ def _get_commands_dict(
def _pop_command_name(argv: list[str]) -> str | None:
- i = 0
- for arg in argv[1:]:
+ for i, arg in enumerate(argv[1:]):
if not arg.startswith("-"):
del argv[i]
return arg
- i += 1
return None
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index f996d1806..61aea3ee4 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -174,13 +174,12 @@ class Command(BaseRunSpiderCommand):
display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
def print_requests(self, lvl: int | None = None, colour: bool = True) -> None:
- if lvl is None:
- if self.requests:
- requests = self.requests[max(self.requests)]
- else:
- requests = []
- else:
+ if lvl is not None:
requests = self.requests.get(lvl, [])
+ elif self.requests:
+ requests = self.requests[max(self.requests)]
+ else:
+ requests = []
print("# Requests ", "-" * 65)
display.pprint(requests, colorize=colour)
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index e0c004580..1adc1530f 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -95,10 +95,7 @@ class Command(ScrapyCommand):
project_name = args[0]
- if len(args) == 2:
- project_dir = Path(args[1])
- else:
- project_dir = Path(args[0])
+ project_dir = Path(args[-1])
if (project_dir / "scrapy.cfg").exists():
self.exitcode = 1
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 9f65794fe..aa8a1a2a4 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -424,10 +424,7 @@ class ScrapyAgent:
headers = TxHeaders(request.headers)
if isinstance(agent, self._TunnelingAgent):
headers.removeHeader(b"Proxy-Authorization")
- if request.body:
- bodyproducer = _RequestBodyProducer(request.body)
- else:
- bodyproducer = None
+ bodyproducer = _RequestBodyProducer(request.body) if request.body else None
start_time = time()
d: Deferred[TxResponse] = agent.request(
method, to_bytes(url, encoding="ascii"), headers, bodyproducer
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 5480df72c..61f444e31 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -291,9 +291,7 @@ class ExecutionEngine:
return False
if self.slot.start_requests is not None: # not all start requests are handled
return False
- if self.slot.scheduler.has_pending_requests():
- return False
- return True
+ return not self.slot.scheduler.has_pending_requests()
def crawl(self, request: Request) -> None:
"""Inject the request into the spider <-> downloader pipeline"""
@@ -388,9 +386,8 @@ class ExecutionEngine:
)
self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
self.spider = spider
- if hasattr(scheduler, "open"):
- if d := scheduler.open(spider):
- yield d
+ if hasattr(scheduler, "open") and (d := scheduler.open(spider)):
+ yield d
yield self.scraper.open_spider(spider)
assert self.crawler.stats
self.crawler.stats.open_spider(spider)
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index a63ee40bf..4b2520aa1 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -198,10 +198,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
# chain, they went through it already from the process_spider_exception method
recovered: MutableChain[_T] | MutableAsyncChain[_T]
last_result_is_async = isinstance(result, AsyncIterable)
- if last_result_is_async:
- recovered = MutableAsyncChain()
- else:
- recovered = MutableChain()
+ recovered = MutableAsyncChain() if last_result_is_async else MutableChain()
# There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async.
# 1. def foo. Sync iterables are passed as is, async ones are downgraded.
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 05af1bf8a..0a28c4549 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import contextlib
import logging
import pprint
import signal
@@ -503,7 +504,6 @@ class CrawlerProcess(CrawlerRunner):
def _stop_reactor(self, _: Any = None) -> None:
from twisted.internet import reactor
- try:
+ # raised if already stopped or in shutdown stage
+ with contextlib.suppress(RuntimeError):
reactor.stop()
- except RuntimeError: # raised if already stopped or in shutdown stage
- pass
diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
index b74140ee1..80107261b 100644
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@@ -42,7 +42,10 @@ class HttpAuthMiddleware:
self, request: Request, spider: Spider
) -> Request | Response | None:
auth = getattr(self, "auth", None)
- if auth and b"Authorization" not in request.headers:
- if not self.domain or url_is_from_any_domain(request.url, [self.domain]):
- request.headers[b"Authorization"] = auth
+ if (
+ auth
+ and b"Authorization" not in request.headers
+ and (not self.domain or url_is_from_any_domain(request.url, [self.domain]))
+ ):
+ request.headers[b"Authorization"] = auth
return None
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 2f3f2db47..cb7fa8c90 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -51,10 +51,7 @@ class HttpProxyMiddleware:
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, "", "", "", ""))
- if user:
- creds = self._basic_auth_header(user, password)
- else:
- creds = None
+ creds = self._basic_auth_header(user, password) if user else None
return creds, proxy_url
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 834a05ae9..46c6aa3fa 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -81,10 +81,7 @@ class BaseItemExporter:
include_empty = self.export_empty_fields
if self.fields_to_export is None:
- if include_empty:
- field_iter = item.field_names()
- else:
- field_iter = item.keys()
+ field_iter = item.field_names() if include_empty else item.keys()
elif isinstance(self.fields_to_export, Mapping):
if include_empty:
field_iter = self.fields_to_export.items()
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index 5ca07394f..afaf81928 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst
from __future__ import annotations
+import contextlib
import logging
import signal
import sys
@@ -69,11 +70,9 @@ class StackTraceDump:
class Debugger:
def __init__(self) -> None:
- try:
+ # win32 platforms don't support SIGUSR signals
+ with contextlib.suppress(AttributeError):
signal.signal(signal.SIGUSR2, self._enter_debugger) # type: ignore[attr-defined]
- except AttributeError:
- # win32 platforms don't support SIGUSR signals
- pass
def _enter_debugger(self, signum: int, frame: FrameType | None) -> None:
assert frame
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index b6e6f55a6..8a3d607b0 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst
from __future__ import annotations
+import contextlib
import logging
import re
import sys
@@ -642,10 +643,8 @@ class FeedExporter:
)
d = {}
for k, v in conf.items():
- try:
+ with contextlib.suppress(NotConfigured):
d[k] = load_object(v)
- except NotConfigured:
- pass
return d
def _exporter_supported(self, format: str) -> bool:
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 929807de8..fe2cbcb86 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -89,10 +89,7 @@ class RFC2616Policy:
return False
cc = self._parse_cachecontrol(request)
# obey user-agent directive "Cache-Control: no-store"
- if b"no-store" in cc:
- return False
- # Any other is eligible for caching
- return True
+ return b"no-store" not in cc
def should_cache_response(self, response: Response, request: Request) -> bool:
# What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py
index 7cf08a1bb..f97577442 100644
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@@ -151,10 +151,7 @@ class PeriodicLog:
return False
if exclude and not include:
return True
- for p in include:
- if p in stat_name:
- return True
- return False
+ return any(p in stat_name for p in include)
def spider_closed(self, spider: Spider, reason: str) -> None:
self.log()
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 60322fe6e..b7c3b9d37 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -64,9 +64,8 @@ class CookieJar:
cookies += self.jar._cookies_for_domain(host, wreq) # type: ignore[attr-defined]
attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined]
- if attrs:
- if not wreq.has_header("Cookie"):
- wreq.add_unredirected_header("Cookie", "; ".join(attrs))
+ if attrs and not wreq.has_header("Cookie"):
+ wreq.add_unredirected_header("Cookie", "; ".join(attrs))
self.processed += 1
if self.processed % self.check_expired_frequency == 0:
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index e5b63ef14..e26cbe05b 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -29,7 +29,7 @@ class JsonRequest(Request):
dumps_kwargs.setdefault("sort_keys", True)
self._dumps_kwargs: dict[str, Any] = dumps_kwargs
- body_passed = kwargs.get("body", None) is not None
+ body_passed = kwargs.get("body") is not None
data: Any = kwargs.pop("data", None)
data_passed: bool = data is not None
@@ -61,7 +61,7 @@ class JsonRequest(Request):
def replace(
self, *args: Any, cls: type[Request] | None = None, **kwargs: Any
) -> Request:
- body_passed = kwargs.get("body", None) is not None
+ body_passed = kwargs.get("body") is not None
data: Any = kwargs.pop("data", None)
data_passed: bool = data is not None
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index f195dbdd7..4fd932b88 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -41,9 +41,12 @@ _collect_string_content = etree.XPath("string()")
def _nons(tag: Any) -> Any:
- if isinstance(tag, str):
- if tag[0] == "{" and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE:
- return tag.split("}")[-1]
+ if (
+ isinstance(tag, str)
+ and tag[0] == "{"
+ and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE
+ ):
+ return tag.split("}")[-1]
return tag
@@ -230,9 +233,7 @@ class LxmlLinkExtractor:
parsed_url, self.deny_extensions
):
return False
- if self.restrict_text and not _matches(link.text, self.restrict_text):
- return False
- return True
+ return not self.restrict_text or _matches(link.text, self.restrict_text)
def matches(self, url: str) -> bool:
if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):
diff --git a/scrapy/mail.py b/scrapy/mail.py
index a3c642401..be2423965 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -111,11 +111,9 @@ class MailSender:
) -> Deferred[None] | None:
from twisted.internet import reactor
- msg: MIMEBase
- if attachs:
- msg = MIMEMultipart()
- else:
- msg = MIMENonMultipart(*mimetype.split("/", 1))
+ msg: MIMEBase = (
+ MIMEMultipart() if attachs else MIMENonMultipart(*mimetype.split("/", 1))
+ )
to = list(arg_to_iter(to))
cc = list(arg_to_iter(cc))
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 16bd45c00..a10117590 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -553,10 +553,8 @@ class FilesPipeline(MediaPipeline):
ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE")
def _get_store(self, uri: str) -> FilesStoreProtocol:
- if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir
- scheme = "file"
- else:
- scheme = urlparse(uri).scheme
+ # to support win32 paths like: C:\\some\dir
+ scheme = "file" if Path(uri).is_absolute() else urlparse(uri).scheme
store_cls = self.STORE_SCHEMES[scheme]
return store_cls(uri)
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 5d0ab1e4d..4a5b9e9cf 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst
from __future__ import annotations
+import contextlib
import os
import signal
from typing import TYPE_CHECKING, Any
@@ -143,12 +144,10 @@ class Shell:
else:
request.meta["handle_httpstatus_all"] = True
response = None
- try:
+ with contextlib.suppress(IgnoreRequest):
response, spider = threads.blockingCallFromThread(
reactor, self._schedule, request, spider
)
- except IgnoreRequest:
- pass
self.populate_vars(response, request, spider)
def populate_vars(
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 18cc991bf..a3a1e5b92 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -360,11 +360,10 @@ class RefererMiddleware:
- otherwise, the policy from settings is used.
"""
policy_name = request.meta.get("referrer_policy")
- if policy_name is None:
- if isinstance(resp_or_url, Response):
- policy_header = resp_or_url.headers.get("Referrer-Policy")
- if policy_header is not None:
- policy_name = to_unicode(policy_header.decode("latin1"))
+ if policy_name is None and isinstance(resp_or_url, Response):
+ policy_header = resp_or_url.headers.get("Referrer-Policy")
+ if policy_header is not None:
+ policy_name = to_unicode(policy_header.decode("latin1"))
if policy_name is None:
return self.default_policy()
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 591737b8e..6b09f36ff 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -1,3 +1,4 @@
+import contextlib
import zlib
from io import BytesIO
from warnings import warn
@@ -37,10 +38,8 @@ else:
return decompressor.process(data)
-try:
+with contextlib.suppress(ImportError):
import zstandard
-except ImportError:
- pass
_CHUNK_SIZE = 65536 # 64 KiB
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 98ecb2f02..3d0e0d3c7 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -8,6 +8,7 @@ This module must not depend on any module outside the Standard Library.
from __future__ import annotations
import collections
+import contextlib
import warnings
import weakref
from collections import OrderedDict
@@ -173,10 +174,9 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
self.data: LocalCache = LocalCache(limit=limit)
def __setitem__(self, key: _KT, value: _VT) -> None:
- try:
+ # if raised, key is not weak-referenceable, skip caching
+ with contextlib.suppress(TypeError):
super().__setitem__(key, value)
- except TypeError:
- pass # key is not weak-referenceable, skip caching
def __getitem__(self, key: _KT) -> _VT | None: # type: ignore[override]
try:
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index c1d3bfffb..5fd176a3f 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -36,7 +36,7 @@ def send_catch_log(
dont_log = named.pop("dont_log", ())
dont_log = tuple(dont_log) if isinstance(dont_log, Sequence) else (dont_log,)
dont_log += (StopDownload,)
- spider = named.get("spider", None)
+ spider = named.get("spider")
responses: list[tuple[TypingAny, TypingAny]] = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
result: TypingAny
@@ -88,7 +88,7 @@ def send_catch_log_deferred(
return failure
dont_log = named.pop("dont_log", None)
- spider = named.get("spider", None)
+ spider = named.get("spider")
dfds: list[Deferred[tuple[TypingAny, TypingAny]]] = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
d: Deferred[TypingAny] = maybeDeferred_coro(
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index d487849bb..db2749d79 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -173,13 +173,19 @@ def strip_url(
parsed_url.username or parsed_url.password
):
netloc = netloc.split("@")[-1]
- if strip_default_port and parsed_url.port:
- if (parsed_url.scheme, parsed_url.port) in (
+
+ if (
+ strip_default_port
+ and parsed_url.port
+ and (parsed_url.scheme, parsed_url.port)
+ in (
("http", 80),
("https", 443),
("ftp", 21),
- ):
- netloc = netloc.replace(f":{parsed_url.port}", "")
+ )
+ ):
+ netloc = netloc.replace(f":{parsed_url.port}", "")
+
return urlunparse(
(
parsed_url.scheme,
diff --git a/tests/test_addons.py b/tests/test_addons.py
index 17949997c..a0caa3511 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -166,19 +166,21 @@ class AddonManagerTest(unittest.TestCase):
def update_settings(self, settings):
pass
- with patch("scrapy.addons.logger") as logger_mock:
- with patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock:
- settings_dict = {
- "ADDONS": {LoggedAddon: 1},
- }
- addon = LoggedAddon()
- build_from_crawler_mock.return_value = addon
- crawler = get_crawler(settings_dict=settings_dict)
- logger_mock.info.assert_called_once_with(
- "Enabled addons:\n%(addons)s",
- {"addons": [addon]},
- extra={"crawler": crawler},
- )
+ with (
+ patch("scrapy.addons.logger") as logger_mock,
+ patch("scrapy.addons.build_from_crawler") as build_from_crawler_mock,
+ ):
+ settings_dict = {
+ "ADDONS": {LoggedAddon: 1},
+ }
+ addon = LoggedAddon()
+ build_from_crawler_mock.return_value = addon
+ crawler = get_crawler(settings_dict=settings_dict)
+ logger_mock.info.assert_called_once_with(
+ "Enabled addons:\n%(addons)s",
+ {"addons": [addon]},
+ extra={"crawler": crawler},
+ )
@inlineCallbacks
def test_enable_addon_in_spider(self):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 05b64e704..0dcbeaec1 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -530,9 +530,10 @@ class Http11TestCase(HttpTestCase):
d = self.download_request(request, Spider("foo"))
def checkDataLoss(failure):
- if failure.check(ResponseFailed):
- if any(r.check(_DataLoss) for r in failure.value.reasons):
- return None
+ if failure.check(ResponseFailed) and any(
+ r.check(_DataLoss) for r in failure.value.reasons
+ ):
+ return None
return failure
d.addCallback(lambda _: self.fail("No DataLoss exception"))
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index b087aaab1..0f149f172 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -756,7 +756,7 @@ class FeedExportTest(FeedExportTestBase):
)
finally:
- for file_path in FEEDS.keys():
+ for file_path in FEEDS:
if not Path(file_path).exists():
continue
@@ -1229,15 +1229,13 @@ class FeedExportTest(FeedExportTestBase):
class CustomFilter2(scrapy.extensions.feedexport.ItemFilter):
def accepts(self, item):
- if "foo" not in item.fields:
- return False
- return True
+ return "foo" in item.fields
class CustomFilter3(scrapy.extensions.feedexport.ItemFilter):
def accepts(self, item):
- if isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1":
- return True
- return False
+ return (
+ isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1"
+ )
formats = {
"json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]',
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 9997b7ab3..c5929c339 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1488,10 +1488,7 @@ def _buildresponse(body, **kwargs):
def _qs(req, encoding="utf-8", to_unicode=False):
- if req.method == "POST":
- qs = req.body
- else:
- qs = req.url.partition("?")[2]
+ qs = req.body if req.method == "POST" else req.url.partition("?")[2]
uqs = unquote_to_bytes(qs)
if to_unicode:
uqs = uqs.decode(encoding)
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index a6c5f0a94..4c3fc36b6 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -634,19 +634,21 @@ class TestGCSFilesStore(unittest.TestCase):
import google.cloud.storage # noqa: F401
except ModuleNotFoundError:
raise unittest.SkipTest("google-cloud-storage is not installed")
- with mock.patch("google.cloud.storage") as _:
- with mock.patch("scrapy.pipelines.files.time") as _:
- uri = "gs://my_bucket/my_prefix/"
- store = GCSFilesStore(uri)
- store.bucket = mock.Mock()
- path = "full/my_data.txt"
- yield store.persist_file(
- path, mock.Mock(), info=None, meta=None, headers=None
- )
- yield store.stat_file(path, info=None)
- expected_blob_path = store.prefix + path
- store.bucket.blob.assert_called_with(expected_blob_path)
- store.bucket.get_blob.assert_called_with(expected_blob_path)
+ with (
+ mock.patch("google.cloud.storage"),
+ mock.patch("scrapy.pipelines.files.time"),
+ ):
+ uri = "gs://my_bucket/my_prefix/"
+ store = GCSFilesStore(uri)
+ store.bucket = mock.Mock()
+ path = "full/my_data.txt"
+ yield store.persist_file(
+ path, mock.Mock(), info=None, meta=None, headers=None
+ )
+ yield store.stat_file(path, info=None)
+ expected_blob_path = store.prefix + path
+ store.bucket.blob.assert_called_with(expected_blob_path)
+ store.bucket.get_blob.assert_called_with(expected_blob_path)
class TestFTPFileStore(unittest.TestCase):
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 503c29e32..96d59c911 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -170,7 +170,7 @@ class BaseSettingsTest(unittest.TestCase):
self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys())
- for key in ctrl_attributes.keys():
+ for key in ctrl_attributes:
attr = self.settings.attributes[key]
ctrl_attr = ctrl_attributes[key]
self.assertEqual(attr.value, ctrl_attr.value)
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 9b53b9b96..d5aac34eb 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -1,3 +1,4 @@
+import contextlib
import shutil
import sys
import tempfile
@@ -22,10 +23,8 @@ module_dir = Path(__file__).resolve().parent
def _copytree(source: Path, target: Path):
- try:
+ with contextlib.suppress(shutil.Error):
shutil.copytree(source, target)
- except shutil.Error:
- pass
class SpiderLoaderTest(unittest.TestCase):
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index eedb6f6af..dc5fbd3c3 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -259,12 +259,14 @@ class WarnWhenSubclassedTest(unittest.TestCase):
self.assertIn("foo.Bar", str(w[1].message))
def test_inspect_stack(self):
- with mock.patch("inspect.stack", side_effect=IndexError):
- with warnings.catch_warnings(record=True) as w:
- DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
+ with (
+ mock.patch("inspect.stack", side_effect=IndexError),
+ warnings.catch_warnings(record=True) as w,
+ ):
+ DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
- class SubClass(DeprecatedName):
- pass
+ class SubClass(DeprecatedName):
+ pass
self.assertIn("Error detecting parent module", str(w[0].message))
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 4c81e3a2f..12507c6a3 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -366,7 +366,7 @@ class UtilsCsvTestCase(unittest.TestCase):
# explicit type check cuz' we no like stinkin' autocasting! yarrr
for result_row in result:
- self.assertTrue(all(isinstance(k, str) for k in result_row.keys()))
+ self.assertTrue(all(isinstance(k, str) for k in result_row))
self.assertTrue(all(isinstance(v, str) for v in result_row.values()))
def test_csviter_delimiter(self):