Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Enable RUF Ruff rules.

Andrey Rakhmatullin838ff99f37d8
diff --git a/pyproject.toml b/pyproject.toml
index ad85e5c75..065382205 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -246,6 +246,8 @@ extend-select = [
"RET",
# flake8-raise
"RSE",
+ # Ruff-specific rules
+ "RUF",
# flake8-bandit
"S",
# flake8-slots
@@ -324,6 +326,14 @@ ignore = [
"PLR2004",
# `for` loop variable overwritten by assignment target
"PLW2901",
+ # String contains ambiguous {}.
+ "RUF001",
+ # Docstring contains ambiguous {}.
+ "RUF002",
+ # Comment contains ambiguous {}.
+ "RUF003",
+ # Mutable class attributes should be annotated with `typing.ClassVar`
+ "RUF012",
# Use of `assert` detected; needed for mypy
"S101",
# FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index c19710a6a..256504c9c 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -13,14 +13,14 @@ from scrapy.selector import Selector
from scrapy.spiders import Spider
__all__ = [
- "__version__",
- "version_info",
- "Spider",
- "Request",
+ "Field",
"FormRequest",
- "Selector",
"Item",
- "Field",
+ "Request",
+ "Selector",
+ "Spider",
+ "__version__",
+ "version_info",
]
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 86d4cc41c..184bd5ca4 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -39,9 +39,8 @@ class Command(BaseRunSpiderCommand):
else:
self.crawler_process.start()
- if (
- self.crawler_process.bootstrap_failed
- or hasattr(self.crawler_process, "has_exception")
+ if self.crawler_process.bootstrap_failed or (
+ hasattr(self.crawler_process, "has_exception")
and self.crawler_process.has_exception
):
self.exitcode = 1
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index cc5c1350b..f996d1806 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -269,7 +269,7 @@ class Command(BaseRunSpiderCommand):
assert self.crawler_process
assert self.spidercls
self.crawler_process.crawl(self.spidercls, **opts.spargs)
- self.pcrawler = list(self.crawler_process.crawlers)[0]
+ self.pcrawler = next(iter(self.crawler_process.crawlers))
self.crawler_process.start()
if not self.first_response:
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index bf8e41020..357ca8b37 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -20,7 +20,7 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType:
if abspath.suffix not in (".py", ".pyw"):
raise ValueError(f"Not a Python source file: {abspath}")
dirname = str(abspath.parent)
- sys.path = [dirname] + sys.path
+ sys.path = [dirname, *sys.path]
try:
module = import_module(abspath.stem)
finally:
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 0b883b43a..612426371 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -101,12 +101,14 @@ class BaseRedirectMiddleware:
if ttl and redirects <= self.max_redirect_times:
redirected.meta["redirect_times"] = redirects
redirected.meta["redirect_ttl"] = ttl - 1
- redirected.meta["redirect_urls"] = request.meta.get("redirect_urls", []) + [
- request.url
+ redirected.meta["redirect_urls"] = [
+ *request.meta.get("redirect_urls", []),
+ request.url,
+ ]
+ redirected.meta["redirect_reasons"] = [
+ *request.meta.get("redirect_reasons", []),
+ reason,
]
- redirected.meta["redirect_reasons"] = request.meta.get(
- "redirect_reasons", []
- ) + [reason]
redirected.dont_filter = request.dont_filter
redirected.priority = request.priority + self.priority_adjust
logger.debug(
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index cdb7ac159..834a05ae9 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -25,13 +25,13 @@ if TYPE_CHECKING:
__all__ = [
"BaseItemExporter",
- "PprintItemExporter",
- "PickleItemExporter",
"CsvItemExporter",
- "XmlItemExporter",
- "JsonLinesItemExporter",
"JsonItemExporter",
+ "JsonLinesItemExporter",
"MarshalItemExporter",
+ "PickleItemExporter",
+ "PprintItemExporter",
+ "XmlItemExporter",
]
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index 6948c394c..5ca07394f 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -77,4 +77,4 @@ class Debugger:
def _enter_debugger(self, signum: int, frame: FrameType | None) -> None:
assert frame
- Pdb().set_trace(frame.f_back) # noqa: T100
+ Pdb().set_trace(frame.f_back)
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 289c60591..e5b63ef14 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -20,7 +20,7 @@ if TYPE_CHECKING:
class JsonRequest(Request):
- attributes: tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
+ attributes: tuple[str, ...] = (*Request.attributes, "dumps_kwargs")
def __init__(
self, *args: Any, dumps_kwargs: dict[str, Any] | None = None, **kwargs: Any
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index f954b5e9e..476f1754e 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -43,7 +43,7 @@ class TextResponse(Response):
_DEFAULT_ENCODING = "ascii"
_cached_decoded_json = _NONE
- attributes: tuple[str, ...] = Response.attributes + ("encoding",)
+ attributes: tuple[str, ...] = (*Response.attributes, "encoding")
def __init__(self, *args: Any, **kwargs: Any):
self._encoding: str | None = kwargs.pop("encoding", None)
diff --git a/scrapy/link.py b/scrapy/link.py
index 1a569f892..9c272ab2f 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -24,7 +24,7 @@ class Link:
of the anchor tag.
"""
- __slots__ = ["url", "text", "fragment", "nofollow"]
+ __slots__ = ["fragment", "nofollow", "text", "url"]
def __init__(
self, url: str, text: str = "", fragment: str = "", nofollow: bool = False
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 544f4adfe..76f9c7856 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -76,8 +76,8 @@ class LogFormatter:
self, request: Request, response: Response, spider: Spider
) -> LogFormatterResult:
"""Logs a message when the crawler finds a webpage."""
- request_flags = f" {str(request.flags)}" if request.flags else ""
- response_flags = f" {str(response.flags)}" if response.flags else ""
+ request_flags = f" {request.flags!s}" if request.flags else ""
+ response_flags = f" {response.flags!s}" if response.flags else ""
return {
"level": logging.DEBUG,
"msg": CRAWLEDMSG,
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 5438b8522..0f3329db1 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -127,8 +127,7 @@ class MediaPipeline(ABC):
if (
not base_class_name
or class_name == base_class_name
- or settings
- and not settings.get(formatted_key)
+ or (settings and not settings.get(formatted_key))
):
return key
return formatted_key
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 93b7fcf17..18cc991bf 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -195,8 +195,7 @@ class StrictOriginPolicy(ReferrerPolicy):
if (
self.tls_protected(response_url)
and self.potentially_trustworthy(request_url)
- or not self.tls_protected(response_url)
- ):
+ ) or not self.tls_protected(response_url):
return self.origin_referrer(response_url)
return None
@@ -249,8 +248,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
if (
self.tls_protected(response_url)
and self.potentially_trustworthy(request_url)
- or not self.tls_protected(response_url)
- ):
+ ) or not self.tls_protected(response_url):
return self.origin_referrer(response_url)
return None
@@ -282,7 +280,7 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
using ``file://`` or ``s3://`` scheme.
"""
- NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3")
+ NOREFERRER_SCHEMES: tuple[str, ...] = (*LOCAL_SCHEMES, "file", "s3")
name: str = POLICY_SCRAPY_DEFAULT
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 5ce4863f6..d319e7950 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -252,7 +252,9 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:
def returns_none(return_node: ast.Return) -> bool:
value = return_node.value
- return value is None or isinstance(value, ast.Constant) and value.value is None
+ return value is None or (
+ isinstance(value, ast.Constant) and value.value is None
+ )
if inspect.isgeneratorfunction(callable):
func = callable
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 66a06a9f0..679e38206 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -100,7 +100,7 @@ def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> No
asyncioreactor.install(eventloop=event_loop)
else:
*module, _ = reactor_path.split(".")
- installer_path = module + ["install"]
+ installer_path = [*module, "install"]
installer = load_object(".".join(installer_path))
with suppress(error.ReactorAlreadyInstalledError):
installer()
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index ad811e804..7f2b178f5 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -229,7 +229,8 @@ def request_to_curl(request: Request) -> str:
cookies = f"--cookie '{cookie}'"
elif isinstance(request.cookies, list):
cookie = "; ".join(
- f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies
+ f"{next(iter(c.keys()))}={next(iter(c.values()))}"
+ for c in request.cookies
)
cookies = f"--cookie '{cookie}'"
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index 05e04e2d1..3b1035eab 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -31,7 +31,7 @@ class ProcessTest:
if settings is not None:
env["SCRAPY_SETTINGS_MODULE"] = settings
assert self.command
- cmd = self.prefix + [self.command] + list(args)
+ cmd = [*self.prefix, self.command, *args]
pp = TestProcessProtocol()
pp.deferred.addCallback(self._process_finished, cmd, check_code)
reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd)
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 3bf831c26..d487849bb 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -51,7 +51,7 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:
def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool:
"""Return True if the url belongs to the given spider"""
return url_is_from_any_domain(
- url, [spider.name] + list(getattr(spider, "allowed_domains", []))
+ url, [spider.name, *getattr(spider, "allowed_domains", [])]
)
diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index ff1f9b346..052321ae3 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -11,7 +11,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings.default_settings import LOG_VERSIONS
from scrapy.utils.ssl import get_openssl_version
-_DEFAULT_SOFTWARE = ["Scrapy"] + LOG_VERSIONS
+_DEFAULT_SOFTWARE = ["Scrapy", *LOG_VERSIONS]
def _version(item):
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 4835e936b..acd524ea4 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -21,7 +21,7 @@ class CmdlineTest(unittest.TestCase):
def _execute(self, *new_args, **kwargs):
encoding = sys.stdout.encoding or "utf-8"
- args = (sys.executable, "-m", "scrapy.cmdline") + new_args
+ args = (sys.executable, "-m", "scrapy.cmdline", *new_args)
proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)
comm = proc.communicate()[0].strip()
return comm.decode(encoding)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 32b69de8a..9d5720b98 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -87,13 +87,13 @@ class ProjectTest(unittest.TestCase):
def call(self, *new_args, **kwargs):
with TemporaryFile() as out:
- args = (sys.executable, "-m", "scrapy.cmdline") + new_args
+ args = (sys.executable, "-m", "scrapy.cmdline", *new_args)
return subprocess.call(
args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs
)
def proc(self, *new_args, **popen_kwargs):
- args = (sys.executable, "-m", "scrapy.cmdline") + new_args
+ args = (sys.executable, "-m", "scrapy.cmdline", *new_args)
p = subprocess.Popen(
args,
cwd=popen_kwargs.pop("cwd", self.cwd),
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index f3e5ebf5d..8b3a6eeca 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -647,7 +647,7 @@ class ScriptRunnerMixin:
def get_script_args(self, script_name: str, *script_args: str) -> list[str]:
script_path = self.script_dir / script_name
- return [sys.executable, str(script_path)] + list(script_args)
+ return [sys.executable, str(script_path), *script_args]
def run_script(self, script_name: str, *script_args: str) -> str:
args = self.get_script_args(script_name, *script_args)
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index c99f19b03..1eb7dcf9d 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -114,7 +114,7 @@ class RetryTest(unittest.TestCase):
def test_exception_to_retry_added(self):
exc = ValueError
settings_dict = {
- "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc],
+ "RETRY_EXCEPTIONS": [*RETRY_EXCEPTIONS, exc],
}
crawler = get_crawler(Spider, settings_dict=settings_dict)
mw = RetryMiddleware.from_crawler(crawler)
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 55f9ecac9..879bc8697 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -31,7 +31,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):
def start_requests(self):
self.times = {None: []}
- slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None]
+ slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None]
for slot in slots:
url = self.mockserver.url(f"/?downloader_slot={slot}")
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index fa9389044..522c6638d 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -116,7 +116,7 @@ class BaseItemExporterTest(unittest.TestCase):
)
ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1")
- _, name = list(ie._get_serialized_fields(self.i))[0]
+ _, name = next(iter(ie._get_serialized_fields(self.i)))
assert isinstance(name, str)
self.assertEqual(name, "John\xa3")
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 679cc8238..0730cff3a 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -960,7 +960,7 @@ class XmlResponseTest(TextResponseTest):
class CustomResponse(TextResponse):
- attributes = TextResponse.attributes + ("foo", "bar")
+ attributes = (*TextResponse.attributes, "foo", "bar")
def __init__(self, *args, **kwargs) -> None:
self.foo = kwargs.pop("foo", None)
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index ef07d625f..58efad585 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -61,11 +61,11 @@ Foo 1 oldest: 0s ago\n\n""",
)
def test_get_oldest(self):
- o1 = Foo() # noqa: F841
+ o1 = Foo()
o1_time = time()
- o2 = Bar() # noqa: F841
+ o2 = Bar()
o3_time = time()
if o3_time <= o1_time:
@@ -80,9 +80,9 @@ Foo 1 oldest: 0s ago\n\n""",
self.assertIsNone(trackref.get_oldest("XXX"))
def test_iter_all(self):
- o1 = Foo() # noqa: F841
+ o1 = Foo()
o2 = Bar() # noqa: F841
- o3 = Foo() # noqa: F841
+ o3 = Foo()
self.assertEqual(
set(trackref.iter_all("Foo")),
{o1, o3},